diff options
author | Arne Juul <arnej@yahoo-inc.com> | 2018-06-20 09:48:02 +0200 |
---|---|---|
committer | Arne Juul <arnej@yahoo-inc.com> | 2018-06-20 09:48:02 +0200 |
commit | 44fc1380b66867958f89c47ac8752926b3787a4d (patch) | |
tree | ea324d387b055c3ccf0921f9f973c95b16bad2e2 | |
parent | 016e584f0ad5a071e13d75eb8ad5ddb46b8c54f5 (diff) | |
parent | 79e7562d34a20ed28621a9ec0dc296eda0881428 (diff) |
Merge branch 'master' into arnej/use-log-abort
428 files changed, 8601 insertions, 4821 deletions
diff --git a/annotations/src/main/resources/.gitignore b/annotations/src/main/resources/.gitignore index b347c417aa1..e69de29bb2d 100644 --- a/annotations/src/main/resources/.gitignore +++ b/annotations/src/main/resources/.gitignore @@ -1 +0,0 @@ -/getversion diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 982cb89f2bf..16e18e99e1b 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -63,38 +63,6 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> </dependency> - <dependency> - <groupId>com.yahoo.athenz</groupId> - <artifactId>athenz-zts-java-client</artifactId> - <scope>compile</scope> - <exclusions> - <!--Exclude all bundles provided by JDisc --> - <exclusion> - <groupId>org.bouncycastle</groupId> - <artifactId>bcpkix-jdk15on</artifactId> - </exclusion> - <exclusion> - <groupId>org.bouncycastle</groupId> - <artifactId>bcprov-jdk15on</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </exclusion> - </exclusions> - </dependency> <!-- TEST --> <dependency> diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java index 2a517e06ae2..2fc696d722b 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java @@ -8,24 +8,35 @@ import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.http.ssl.SslKeyStoreConfigurator; import com.yahoo.jdisc.http.ssl.SslKeyStoreContext; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; +import com.yahoo.vespa.athenz.client.zts.Identity; +import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.KeyStoreBuilder; import com.yahoo.vespa.athenz.tls.KeyStoreType; +import com.yahoo.vespa.athenz.tls.KeyUtils; +import com.yahoo.vespa.athenz.tls.X509CertificateUtils; +import com.yahoo.vespa.athenz.utils.SiaUtils; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.AthenzCertificateClient; import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.GeneralSecurityException; +import java.security.KeyPair; import java.security.KeyStore; import java.security.KeyStoreException; import java.security.PrivateKey; +import java.security.PublicKey; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.Executors; @@ -33,7 +44,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; -import static com.yahoo.vespa.athenz.tls.KeyStoreUtils.writeKeyStoreToFile; import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils.getZoneConfig; /** @@ -47,13 +57,15 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements private static final Logger log = Logger.getLogger(AthenzSslKeyStoreConfigurator.class.getName()); private static final String CERTIFICATE_ALIAS = "athenz"; private static final Duration EXPIRATION_MARGIN = Duration.ofHours(6); + private static final Path VESPA_SIA_DIRECTORY = Paths.get(Defaults.getDefaults().underVespaHome("var/vespa/sia")); + private static final Path CA_CERT_FILE = VESPA_SIA_DIRECTORY.resolve("ca-certs.pem"); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); - private final AthenzCertificateClient certificateClient; + private final ZtsClient ztsClient; private final KeyProvider keyProvider; private final AthenzProviderServiceConfig.Zones zoneConfig; private final Duration updatePeriod; - private final Path keystoreDirectory; + private final AthenzService configserverIdentity; private volatile KeyStoreAndPassword currentKeyStore; @Inject @@ -63,55 +75,51 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements Zone zone, ConfigserverConfig configserverConfig) { AthenzProviderServiceConfig.Zones zoneConfig = getZoneConfig(config, zone); - Path keystoreDirectory = createKeystoreCacheDirectory(configserverConfig); - AthenzCertificateClient certificateClient = new AthenzCertificateClient(bootstrapIdentity, zoneConfig); + AthenzService configserverIdentity = new AthenzService(zoneConfig.domain(), zoneConfig.serviceName()); Duration updatePeriod = Duration.ofDays(config.updatePeriodDays()); - this.certificateClient = certificateClient; + DefaultZtsClient ztsClient = new DefaultZtsClient(URI.create(zoneConfig.ztsUrl()).resolve("/zts/v1"), bootstrapIdentity); // TODO Remove URI.resolve() once config in hosted is updated + this.ztsClient = ztsClient; this.keyProvider = keyProvider; this.zoneConfig = zoneConfig; - this.currentKeyStore = initializeKeystore(keyProvider, certificateClient, zoneConfig, keystoreDirectory, updatePeriod); + this.currentKeyStore = initializeKeystore(configserverIdentity, keyProvider, ztsClient, zoneConfig, updatePeriod); this.updatePeriod = updatePeriod; - this.keystoreDirectory = keystoreDirectory; + this.configserverIdentity = configserverIdentity; } - private static KeyStoreAndPassword initializeKeystore(KeyProvider keyProvider, - AthenzCertificateClient certificateClient, - AthenzProviderServiceConfig.Zones zoneConfig, - Path keystoreCacheDirectory, - Duration updatePeriod) { - return tryReadKeystoreFile(keystoreCacheDirectory, updatePeriod) - .orElseGet(() -> downloadCertificate(keyProvider, certificateClient, zoneConfig, keystoreCacheDirectory)); + private static KeyStoreAndPassword initializeKeystore(AthenzService configserverIdentity, + KeyProvider keyProvider, + ZtsClient ztsClient, + AthenzProviderServiceConfig.Zones keystoreCacheDirectory, + Duration updatePeriod) { + return tryReadKeystoreFile(configserverIdentity, updatePeriod) + .orElseGet(() -> downloadCertificate(configserverIdentity, keyProvider, ztsClient, keystoreCacheDirectory)); } - private static Optional<KeyStoreAndPassword> tryReadKeystoreFile(Path keystoreDirectory, Duration updatePeriod) { + private static Optional<KeyStoreAndPassword> tryReadKeystoreFile(AthenzService configserverIdentity, + Duration updatePeriod) { try { - Path keystoreFile = keystoreFile(keystoreDirectory); - Path keystoreSecretFile = keystoreSecretFile(keystoreDirectory); - if (!Files.exists(keystoreFile) || !Files.exists(keystoreSecretFile)) return Optional.empty(); - KeyStore keyStore = KeyStoreBuilder.withType(KeyStoreType.JKS) - .fromFile(keystoreFile.toFile()) - .build(); + Optional<X509Certificate> certificate = SiaUtils.readCertificateFile(VESPA_SIA_DIRECTORY, configserverIdentity); + if (!certificate.isPresent()) return Optional.empty(); + Optional<PrivateKey> privateKey = SiaUtils.readPrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity); + if (!privateKey.isPresent()) return Optional.empty(); Instant minimumExpiration = Instant.now().plus(updatePeriod).plus(EXPIRATION_MARGIN); - boolean isExpired = getCertificateExpiry(keyStore).isBefore(minimumExpiration); + boolean isExpired = certificate.get().getNotAfter().toInstant().isBefore(minimumExpiration); if (isExpired) return Optional.empty(); - char[] pwd = new String(Files.readAllBytes(keystoreSecretFile)).toCharArray(); - return Optional.of(new KeyStoreAndPassword(keyStore, pwd)); - } catch (GeneralSecurityException | IOException e) { - log.log(LogLevel.ERROR, "Failed to read keystore from disk: " + e.getMessage(), e); - return Optional.empty(); - } - } + if (Files.notExists(CA_CERT_FILE)) return Optional.empty(); + List<X509Certificate> caCertificates = X509CertificateUtils.certificateListFromPem(new String(Files.readAllBytes(CA_CERT_FILE))); - private static Path createKeystoreCacheDirectory(ConfigserverConfig configserverConfig) { - return Paths.get(Defaults.getDefaults().underVespaHome(configserverConfig.configServerDBDir())); - } + List<X509Certificate> chain = new ArrayList<>(); + chain.add(certificate.get()); + chain.addAll(caCertificates); - private static Path keystoreFile(Path keystoreDirectory) { - return keystoreDirectory.resolve("server-x509-athenz-cert.jks"); - } - - private static Path keystoreSecretFile(Path keystoreDirectory) { - return keystoreDirectory.resolve("keystore-secret"); + char[] password = generateKeystorePassword(); + KeyStore keyStore = KeyStoreBuilder.withType(KeyStoreType.JKS) + .withKeyEntry(CERTIFICATE_ALIAS, privateKey.get(), password, chain) + .build(); + return Optional.of(new KeyStoreAndPassword(keyStore, password)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } @Override @@ -128,6 +136,7 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements try { scheduler.shutdownNow(); scheduler.awaitTermination(30, TimeUnit.SECONDS); + ztsClient.close(); } catch (InterruptedException e) { throw new RuntimeException("Failed to shutdown Athenz certificate updater on time", e); } @@ -142,31 +151,42 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements return certificate.getNotAfter().toInstant(); } - private static KeyStoreAndPassword downloadCertificate(KeyProvider keyProvider, - AthenzCertificateClient certificateClient, - AthenzProviderServiceConfig.Zones zoneConfig, - Path keystoreDirectory) { + private static KeyStoreAndPassword downloadCertificate(AthenzService configserverIdentity, + KeyProvider keyProvider, + ZtsClient ztsClient, + AthenzProviderServiceConfig.Zones zoneConfig) { PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); - X509Certificate certificate = certificateClient.updateCertificate(privateKey); + PublicKey publicKey = KeyUtils.extractPublicKey(privateKey); + Identity serviceIdentity = ztsClient.getServiceIdentity(configserverIdentity, + Integer.toString(zoneConfig.secretVersion()), + new KeyPair(publicKey, privateKey), + zoneConfig.certDnsSuffix()); + X509Certificate certificate = serviceIdentity.certificate(); + writeCredentials(configserverIdentity, certificate, serviceIdentity.caCertificates(), privateKey); Instant expirationTime = certificate.getNotAfter().toInstant(); Duration expiry = Duration.between(certificate.getNotBefore().toInstant(), expirationTime); log.log(LogLevel.INFO, String.format("Got Athenz x509 certificate with expiry %s (expires %s)", expiry, expirationTime)); + List<X509Certificate> chain = new ArrayList<>(); + chain.add(certificate); + chain.addAll(serviceIdentity.caCertificates()); char[] keystorePassword = generateKeystorePassword(); KeyStore keyStore = KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry(CERTIFICATE_ALIAS, privateKey, keystorePassword, certificate) + .withKeyEntry(CERTIFICATE_ALIAS, privateKey, keystorePassword, chain) .build(); - KeyStoreAndPassword keyStoreAndPassword = new KeyStoreAndPassword(keyStore, keystorePassword); - tryWriteKeystore(keyStoreAndPassword, keystoreDirectory); - return keyStoreAndPassword; + return new KeyStoreAndPassword(keyStore, keystorePassword); } - private static void tryWriteKeystore(KeyStoreAndPassword keyStore, Path keystoreDirectory) { - try { - writeKeyStoreToFile(keyStore.keyStore, keystoreFile(keystoreDirectory).toFile()); - Files.write(keystoreSecretFile(keystoreDirectory), new String(keyStore.password).getBytes()); - } catch (Exception e) { - log.log(LogLevel.ERROR, "Failed to write keystore to disk: " + e.getMessage(), e); + private static void writeCredentials(AthenzService configserverIdentity, + X509Certificate certificate, + List<X509Certificate> caCertificates, + PrivateKey privateKey) { + SiaUtils.writeCertificateFile(VESPA_SIA_DIRECTORY, configserverIdentity, certificate); + SiaUtils.writePrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity, privateKey); + try { + Files.write(CA_CERT_FILE, X509CertificateUtils.toPem(caCertificates).getBytes()); + } catch (IOException e) { + throw new UncheckedIOException(e); } } @@ -186,7 +206,7 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements public void run() { try { log.log(LogLevel.INFO, "Updating Athenz certificate from ZTS"); - currentKeyStore = downloadCertificate(keyProvider, certificateClient, zoneConfig, keystoreDirectory); + currentKeyStore = downloadCertificate(configserverIdentity, keyProvider, ztsClient, zoneConfig); sslKeyStoreContext.updateKeyStore(currentKeyStore.keyStore, new String(currentKeyStore.password)); log.log(LogLevel.INFO, "Athenz certificate reload successfully completed"); } catch (Throwable e) { diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java index 5fff85f695d..0abbb5a64f5 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java @@ -5,11 +5,11 @@ import com.google.inject.Inject; import com.yahoo.config.provision.Zone; import com.yahoo.net.HostName; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; +import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils; @@ -19,9 +19,7 @@ import com.yahoo.vespa.hosted.provision.node.Allocation; import java.net.URI; import java.security.PrivateKey; -import java.security.Signature; import java.time.Instant; -import java.util.Base64; import java.util.HashSet; import java.util.Set; @@ -33,6 +31,7 @@ import java.util.Set; */ public class IdentityDocumentGenerator { + private final IdentityDocumentSigner signer = new IdentityDocumentSigner(); private final NodeRepository nodeRepository; private final Zone zone; private final KeyProvider keyProvider; @@ -53,16 +52,13 @@ public class IdentityDocumentGenerator { Node node = nodeRepository.getNode(hostname).orElseThrow(() -> new RuntimeException("Unable to find node " + hostname)); try { IdentityDocument identityDocument = generateIdDocument(node, identityType); - String identityDocumentString = Utils.getMapper().writeValueAsString(EntityBindingsMapper.toIdentityDocumentEntity(identityDocument)); - - String encodedIdentityDocument = - Base64.getEncoder().encodeToString(identityDocumentString.getBytes()); - Signature sigGenerator = Signature.getInstance("SHA512withRSA"); PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); - sigGenerator.initSign(privateKey); - sigGenerator.update(encodedIdentityDocument.getBytes()); - String signature = Base64.getEncoder().encodeToString(sigGenerator.sign()); + AthenzService providerService = new AthenzService(zoneConfig.domain(), zoneConfig.serviceName()); + + String signature = signer.generateSignature( + identityDocument.providerUniqueId(), providerService, identityDocument.configServerHostname(), + identityDocument.instanceHostname(), identityDocument.createdAt(), identityDocument.ipAddresses(), identityType, privateKey); return new SignedIdentityDocument( identityDocument, @@ -70,7 +66,7 @@ public class IdentityDocumentGenerator { SignedIdentityDocument.DEFAULT_KEY_VERSION, identityDocument.providerUniqueId(), toZoneDnsSuffix(zone, zoneConfig.certDnsSuffix()), - new AthenzService(zoneConfig.domain(), zoneConfig.serviceName()), + providerService, URI.create(zoneConfig.ztsUrl()), SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION, identityDocument.configServerHostname(), diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/AthenzCertificateClient.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/AthenzCertificateClient.java deleted file mode 100644 index 193a573c98d..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/AthenzCertificateClient.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; - -import com.yahoo.athenz.zts.InstanceRefreshRequest; -import com.yahoo.athenz.zts.ZTSClient; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.tls.X509CertificateUtils; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; - -import javax.net.ssl.SSLContext; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; - -/** - * @author bjorncs - */ -public class AthenzCertificateClient { - - private final AthenzProviderServiceConfig.Zones zoneConfig; - private final ServiceIdentityProvider bootstrapIdentity; - - public AthenzCertificateClient(ServiceIdentityProvider bootstrapIdentity, - AthenzProviderServiceConfig.Zones zoneConfig) { - this.bootstrapIdentity = bootstrapIdentity; - this.zoneConfig = zoneConfig; - } - - public X509Certificate updateCertificate(PrivateKey privateKey) { - SSLContext bootstrapSslContext = bootstrapIdentity.getIdentitySslContext(); - ZTSClient ztsClient = new ZTSClient(zoneConfig.ztsUrl(), bootstrapSslContext); - InstanceRefreshRequest req = - ZTSClient.generateInstanceRefreshRequest( - zoneConfig.domain(), zoneConfig.serviceName(), privateKey, zoneConfig.certDnsSuffix(), /*expiryTime*/0); - req.setKeyId(Integer.toString(zoneConfig.secretVersion())); - String pemEncoded = ztsClient.postInstanceRefreshRequest(zoneConfig.domain(), zoneConfig.serviceName(), req) - .getCertificate(); - return X509CertificateUtils.fromPem(pemEncoded); - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index 0201c46b253..b75f7d05394 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -11,15 +11,10 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; +import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; import java.security.PublicKey; -import java.security.Signature; -import java.security.SignatureException; -import java.util.Base64; import java.util.Optional; import java.util.logging.Logger; @@ -35,6 +30,7 @@ public class InstanceValidator { static final String SERVICE_PROPERTIES_DOMAIN_KEY = "identity.domain"; static final String SERVICE_PROPERTIES_SERVICE_KEY = "identity.service"; + private final IdentityDocumentSigner signer = new IdentityDocumentSigner(); private final KeyProvider keyProvider; private final SuperModelProvider superModelProvider; @@ -55,7 +51,9 @@ public class InstanceValidator { } log.log(LogLevel.INFO, () -> String.format("Validating instance %s.", providerUniqueId)); - if (isInstanceSignatureValid(instanceConfirmation)) { + + PublicKey publicKey = keyProvider.getPublicKey(signedIdentityDocument.signingKeyVersion()); + if (signer.hasValidSignature(signedIdentityDocument, publicKey)) { log.log(LogLevel.INFO, () -> String.format("Instance %s is valid.", providerUniqueId)); return true; } @@ -63,24 +61,6 @@ public class InstanceValidator { return false; } - boolean isInstanceSignatureValid(InstanceConfirmation instanceConfirmation) { - SignedIdentityDocumentEntity signedIdentityDocument = instanceConfirmation.signedIdentityDocument; - - PublicKey publicKey = keyProvider.getPublicKey(signedIdentityDocument.signingKeyVersion); - return isSignatureValid(publicKey, signedIdentityDocument.rawIdentityDocument, signedIdentityDocument.signature); - } - - public static boolean isSignatureValid(PublicKey publicKey, String rawIdentityDocument, String signature) { - try { - Signature signatureVerifier = Signature.getInstance("SHA512withRSA"); - signatureVerifier.initVerify(publicKey); - signatureVerifier.update(rawIdentityDocument.getBytes()); - return signatureVerifier.verify(Base64.getDecoder().decode(signature)); - } catch (NoSuchAlgorithmException | InvalidKeyException | SignatureException e) { - throw new RuntimeException(e); - } - } - // If/when we dont care about logging exactly whats wrong, this can be simplified // TODO Use identity type to determine if this check should be performed boolean isSameIdentityAsInServicesXml(ApplicationId applicationId, String domain, String service) { diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java index 078ef1b7e39..a1839ec62a2 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java @@ -14,14 +14,12 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; +import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.AutoGeneratedKeyProvider; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -48,7 +46,7 @@ public class IdentityDocumentGeneratorTest { private static final Zone ZONE = new Zone(SystemName.cd, Environment.dev, RegionName.from("us-north-1")); @Test - public void generates_valid_identity_document() throws Exception { + public void generates_valid_identity_document() { String parentHostname = "docker-host"; String containerHostname = "docker-container"; @@ -85,7 +83,7 @@ public class IdentityDocumentGeneratorTest { SignedIdentityDocument signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(containerHostname, IdentityType.TENANT); // Verify attributes - assertEquals(containerHostname, signedIdentityDocument.identityDocument().instanceHostname()); + assertEquals(containerHostname, signedIdentityDocument.instanceHostname()); String environment = "dev"; String region = "us-north-1"; @@ -97,14 +95,11 @@ public class IdentityDocumentGeneratorTest { assertEquals(expectedProviderUniqueId, signedIdentityDocument.providerUniqueId()); // Validate that container ips are present - assertThat(signedIdentityDocument.identityDocument().ipAddresses(), Matchers.containsInAnyOrder("::1")); + assertThat(signedIdentityDocument.ipAddresses(), Matchers.containsInAnyOrder("::1")); - SignedIdentityDocumentEntity signedIdentityDocumentEntity = EntityBindingsMapper.toSignedIdentityDocumentEntity(signedIdentityDocument); + IdentityDocumentSigner signer = new IdentityDocumentSigner(); // Validate signature - assertTrue("Message", InstanceValidator.isSignatureValid(keyProvider.getPublicKey(0), - signedIdentityDocumentEntity.rawIdentityDocument, - signedIdentityDocument.signature())); - + assertTrue(signer.hasValidSignature(signedIdentityDocument, keyProvider.getPublicKey(0))); } } diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java index 54411b424eb..04c4d4da51a 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java @@ -14,8 +14,6 @@ import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentEntity; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.identityprovider.api.bindings.VespaUniqueInstanceIdEntity; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.AutoGeneratedKeyProvider; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils; import org.junit.Test; @@ -49,27 +47,6 @@ public class InstanceValidatorTest { private final String domain = "domain"; private final String service = "service"; - @Test - public void valid_signature() throws Exception { - KeyProvider keyProvider = new AutoGeneratedKeyProvider(); - InstanceValidator instanceValidator = new InstanceValidator(keyProvider, null); - InstanceConfirmation instanceConfirmation = createInstanceConfirmation( - keyProvider.getPrivateKey(0), applicationId, domain, service); - - assertTrue(instanceValidator.isInstanceSignatureValid(instanceConfirmation)); - } - - @Test - public void invalid_signature() throws Exception { - KeyProvider keyProvider = new AutoGeneratedKeyProvider(); - InstanceValidator instanceValidator = new InstanceValidator(keyProvider, null); - - KeyProvider fakeKeyProvider = new AutoGeneratedKeyProvider(); - InstanceConfirmation instanceConfirmation = createInstanceConfirmation( - fakeKeyProvider.getPrivateKey(0), applicationId, domain, service); - - assertFalse(instanceValidator.isInstanceSignatureValid(instanceConfirmation)); - } @Test public void application_does_not_exist() { diff --git a/bundle-plugin-test/pom.xml b/bundle-plugin-test/pom.xml index 53be71352c8..4b36435529e 100644 --- a/bundle-plugin-test/pom.xml +++ b/bundle-plugin-test/pom.xml @@ -44,36 +44,15 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - <scope>provided</scope> - </dependency> - - <dependency> <!-- Added to verify that module-info.class can be handled by bundle-plugin without throwing an exception. --> <groupId>javax.xml.bind</groupId> <artifactId>jaxb-api</artifactId> <version>2.3.0</version> </dependency> - </dependencies> <build> <plugins> <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>add-source</goal> - <goal>compile</goal> - <goal>testCompile</goal> - </goals> - </execution> - </executions> - </plugin> - - <plugin> <groupId>com.yahoo.vespa</groupId> <artifactId>bundle-plugin</artifactId> <version>${project.version}</version> diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index b68ffa811b2..a5cb3c9fbcc 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -39,16 +39,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - </dependency> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parser-combinators_${scala.major-version}</artifactId> - </dependency> - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.major-version}</artifactId> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-library</artifactId> <scope>test</scope> </dependency> <dependency> @@ -95,19 +87,6 @@ <artifactId>maven-compiler-plugin</artifactId> </plugin> <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>add-source</goal> - <goal>compile</goal> - <goal>testCompile</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-plugin-plugin</artifactId> </plugin> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java new file mode 100644 index 00000000000..798fea2644e --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java @@ -0,0 +1,94 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.bundle; + +import com.yahoo.container.plugin.osgi.ExportPackageParser; +import com.yahoo.container.plugin.osgi.ExportPackages.Export; +import com.yahoo.container.plugin.util.JarFiles; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.jar.Manifest; +import java.util.stream.Collectors; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class AnalyzeBundle { + public static class PublicPackages { + public final List<Export> exports; + public final List<String> globals; + + public PublicPackages(List<Export> exports, List<String> globals) { + this.exports = exports; + this.globals = globals; + } + } + + public static PublicPackages publicPackagesAggregated(Collection<File> jarFiles) { + List<Export> exports = new ArrayList<>(); + List<String> globals = new ArrayList<>(); + + for (File jarFile : jarFiles) { + PublicPackages pp = publicPackages(jarFile); + exports.addAll(pp.exports); + globals.addAll(pp.globals); + } + return new PublicPackages(exports, globals); + } + + public static PublicPackages publicPackages(File jarFile) { + try { + Optional<Manifest> jarManifest = JarFiles.getManifest(jarFile); + if (jarManifest.isPresent()) { + Manifest manifest = jarManifest.get(); + if (isOsgiManifest(manifest)) { + return new PublicPackages(parseExports(manifest), parseGlobals(manifest)); + } + } + return new PublicPackages(Collections.emptyList(), Collections.emptyList()); + } catch (Exception e) { + throw new RuntimeException(String.format("Invalid manifest in bundle '%s'", jarFile.getPath()), e); + } + } + + public static Optional<String> bundleSymbolicName(File jarFile) { + return JarFiles.getManifest(jarFile).flatMap(AnalyzeBundle::getBundleSymbolicName); + } + + private static List<Export> parseExportsFromAttribute(Manifest manifest, String attributeName) { + return getMainAttributeValue(manifest, attributeName).map(ExportPackageParser::parseExports).orElseGet(() -> new ArrayList<>()); + } + + private static List<Export> parseExports(Manifest jarManifest) { + return parseExportsFromAttribute(jarManifest, "Export-Package"); + } + + private static List<String> parseGlobals(Manifest manifest) { + List<Export> globals = parseExportsFromAttribute(manifest, "Global-Package"); + + for (Export export : globals) { + if (export.getParameters().isEmpty() == false) { + throw new RuntimeException("Parameters not valid for Global-Package."); + } + } + + return globals.stream().flatMap(g -> g.getPackageNames().stream()).collect(Collectors.toList()); + } + + private static Optional<String> getMainAttributeValue(Manifest manifest, String attributeName) { + return Optional.ofNullable(manifest.getMainAttributes().getValue(attributeName)); + } + + private static boolean isOsgiManifest(Manifest mf) { + return getBundleSymbolicName(mf).isPresent(); + } + + private static Optional<String> getBundleSymbolicName(Manifest mf) { + return getMainAttributeValue(mf, "Bundle-SymbolicName"); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/TransformExportPackages.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/TransformExportPackages.java new file mode 100644 index 00000000000..8686fef0a55 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/TransformExportPackages.java @@ -0,0 +1,62 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.bundle; + +import com.yahoo.container.plugin.osgi.ExportPackages.Export; +import com.yahoo.container.plugin.osgi.ExportPackages.Parameter; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class TransformExportPackages { + public static List<Export> replaceVersions(List<Export> exports, String newVersion) { + List<Export> ret = new ArrayList<>(); + + for (Export export : exports) { + List<Parameter> newParams = new ArrayList<>(); + for (Parameter param : export.getParameters()) { + if ("version".equals(param.getName())) { + newParams.add(new Parameter("version", newVersion)); + } else { + newParams.add(param); + } + } + ret.add(new Export(export.getPackageNames(), newParams)); + } + return ret; + } + + public static List<Export> removeUses(List<Export> exports) { + List<Export> ret = new ArrayList<>(); + + for (Export export : exports) { + List<Parameter> newParams = new ArrayList<>(); + for (Parameter param : export.getParameters()) { + if ("uses".equals(param.getName()) == false) { + newParams.add(param); + } + } + ret.add(new Export(export.getPackageNames(), newParams)); + } + return ret; + } + + public static String toExportPackageProperty(List<Export> exports) { + return exports.stream().map(exp -> { + String oneExport = String.join(";", exp.getPackageNames()); + if (exp.getParameters().size() > 0) { + String paramString = exp.getParameters().stream().map(param -> param.getName() + "=" + quote(param.getValue())).collect(Collectors.joining(";")); + oneExport += ";" + paramString; + } + return oneExport; + }).collect(Collectors.joining(",")); + } + + public static String quote(String s) { + return "\"" + s + "\""; + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java new file mode 100644 index 00000000000..c59f8559405 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java @@ -0,0 +1,87 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import org.objectweb.asm.AnnotationVisitor; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; + +import static com.yahoo.container.plugin.util.IO.withFileInputStream; + +/** + * Main entry point for class analysis + * + * @author Tony Vaagenes + * @author ollivir + */ +public class Analyze { + public static ClassFileMetaData analyzeClass(File classFile) { + try { + return withFileInputStream(classFile, Analyze::analyzeClass); + } catch (RuntimeException e) { + throw new RuntimeException("An error occurred when analyzing " + classFile.getPath(), e); + } + } + + public static ClassFileMetaData analyzeClass(InputStream inputStream) { + try { + AnalyzeClassVisitor visitor = new AnalyzeClassVisitor(); + new ClassReader(inputStream).accept(visitor, ClassReader.SKIP_DEBUG); + return visitor.result(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + static Optional<String> internalNameToClassName(String internalClassName) { + if(internalClassName == null) { + return Optional.empty(); + } else { + return getClassName(Type.getObjectType(internalClassName)); + } + } + + static Optional<String> getClassName(Type aType) { + switch (aType.getSort()) { + case Type.ARRAY: + return getClassName(aType.getElementType()); + case Type.OBJECT: + return Optional.of(aType.getClassName()); + default: + return Optional.empty(); + } + } + + static AnnotationVisitor visitAnnotationDefault(ImportCollector collector) { + return new AnnotationVisitor(Opcodes.ASM6) { + @Override + public void visit(String name, Object value) { + } + + @Override + public void visitEnum(String name, String desc, String value) { + collector.addImportWithTypeDesc(desc); + } + + @Override + public AnnotationVisitor visitArray(String name) { + return this; + } + + @Override + public AnnotationVisitor visitAnnotation(String name, String desc) { + collector.addImportWithTypeDesc(desc); + return this; + } + + @Override + public void visitEnd() { + } + }; + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java new file mode 100644 index 00000000000..d9519fd7986 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java @@ -0,0 +1,162 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; +import org.objectweb.asm.AnnotationVisitor; +import org.objectweb.asm.Attribute; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.FieldVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +/** + * Picks up classes used in class files. + * + * @author Tony Vaagenes + * @author ollivir + */ +class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { + private String name = null; + private Set<String> imports = new HashSet<>(); + private Optional<ExportPackageAnnotation> exportPackageAnnotation = Optional.empty(); + + AnalyzeClassVisitor() { + super(Opcodes.ASM6); + } + + @Override + public Set<String> imports() { + return imports; + } + + @Override + public void visitAttribute(Attribute attribute) { + addImport(Type.getObjectType(attribute.type)); + } + + @Override + public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { + Analyze.getClassName(Type.getReturnType(desc)).ifPresent(imports::add); + Arrays.asList(Type.getArgumentTypes(desc)).forEach(argType -> Analyze.getClassName(argType).ifPresent(imports::add)); + if (exceptions != null) { + Arrays.asList(exceptions).forEach(ex -> Analyze.internalNameToClassName(ex).ifPresent(imports::add)); + } + + AnalyzeSignatureVisitor.analyzeMethod(signature, this); + return new AnalyzeMethodVisitor(this); + } + + @Override + public FieldVisitor visitField(int access, String name, String desc, String signature, Object value) { + Analyze.getClassName(Type.getType(desc)).ifPresent(imports::add); + + AnalyzeSignatureVisitor.analyzeField(signature, this); + return new AnalyzeFieldVisitor(this); + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + this.name = Analyze.internalNameToClassName(name) + .orElseThrow(() -> new RuntimeException("Unable to resolve class name for " + name)); + + addImportWithInternalName(superName); + Arrays.asList(interfaces).forEach(this::addImportWithInternalName); + + AnalyzeSignatureVisitor.analyzeClass(signature, this); + } + + @Override + public void visitInnerClass(String name, String outerName, String innerName, int access) { + } + + @Override + public void visitOuterClass(String owner, String name, String desc) { + } + + @Override + public void visitSource(String source, String debug) { + } + + @Override + public void visitEnd() { + } + + @SuppressWarnings("unchecked") + private static <T> T defaultVersionValue(String name) { + try { + return (T) Version.class.getMethod(name).getDefaultValue(); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException("Could not locate method " + name); + } + } + + private AnnotationVisitor visitExportPackage() { + return new AnnotationVisitor(Opcodes.ASM6) { + private int major = defaultVersionValue("major"); + private int minor = defaultVersionValue("minor"); + private int micro = defaultVersionValue("micro"); + private String qualifier = defaultVersionValue("qualifier"); + + @Override + public void visit(String name, Object value) { + if (name != null) { + switch (name) { + case "major": + major = (int) value; + break; + case "minor": + minor = (int) value; + break; + case "micro": + micro = (int) value; + break; + case "qualifier": + qualifier = (String) value; + break; + } + } + } + + @Override + public void visitEnd() { + exportPackageAnnotation = Optional.of(new ExportPackageAnnotation(major, minor, micro, qualifier)); + } + + @Override + public void visitEnum(String name, String desc, String value) { + } + + @Override + public AnnotationVisitor visitArray(String name) { + return this; + } + + @Override + public AnnotationVisitor visitAnnotation(String name, String desc) { + return this; + } + }; + } + + @Override + public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + if (ExportPackage.class.getName().equals(Type.getType(desc).getClassName())) { + return visitExportPackage(); + } else { + addImportWithTypeDesc(desc); + return Analyze.visitAnnotationDefault(this); + } + } + + ClassFileMetaData result() { + assert (!imports.contains("int")); + return new ClassFileMetaData(name, imports, exportPackageAnnotation); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java new file mode 100644 index 00000000000..ea10b6ef0aa --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java @@ -0,0 +1,49 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import org.objectweb.asm.AnnotationVisitor; +import org.objectweb.asm.Attribute; +import org.objectweb.asm.FieldVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author ollivir + */ +public class AnalyzeFieldVisitor extends FieldVisitor implements ImportCollector { + private final AnalyzeClassVisitor analyzeClassVisitor; + private final Set<String> imports = new HashSet<>(); + + public AnalyzeFieldVisitor(AnalyzeClassVisitor analyzeClassVisitor) { + super(Opcodes.ASM6); + this.analyzeClassVisitor = analyzeClassVisitor; + } + + @Override + public Set<String> imports() { + return imports; + } + + @Override + public void visitAttribute(Attribute attribute) { + addImport(Type.getObjectType(attribute.type)); + } + + public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + addImportWithTypeDesc(desc); + + return Analyze.visitAnnotationDefault(this); + } + + public AnnotationVisitor visitParameterAnnotation(int parameter, String desc, boolean visible) { + return visitAnnotation(desc, visible); + } + + @Override + public void visitEnd() { + analyzeClassVisitor.addImports(imports); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java new file mode 100644 index 00000000000..b1a92f9c10b --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java @@ -0,0 +1,168 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import org.objectweb.asm.AnnotationVisitor; +import org.objectweb.asm.Attribute; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * Picks up classes used in method bodies. + * + * @author Tony Vaagenes + * @author ollivir + */ +class AnalyzeMethodVisitor extends MethodVisitor implements ImportCollector { + private final Set<String> imports = new HashSet<>(); + private final AnalyzeClassVisitor analyzeClassVisitor; + + AnalyzeMethodVisitor(AnalyzeClassVisitor analyzeClassVisitor) { + super(Opcodes.ASM6); + this.analyzeClassVisitor = analyzeClassVisitor; + } + + public Set<String> imports() { + return imports; + } + + @Override + public AnnotationVisitor visitParameterAnnotation(int parameter, String desc, boolean visible) { + return visitAnnotation(desc, visible); + } + + @Override + public AnnotationVisitor visitAnnotationDefault() { + return Analyze.visitAnnotationDefault(this); + } + + @Override + public void visitAttribute(Attribute attribute) { + addImport(Type.getObjectType(attribute.type)); + } + + @Override + public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + addImportWithTypeDesc(desc); + + return Analyze.visitAnnotationDefault(this); + } + + @Override + public void visitEnd() { + super.visitEnd(); + analyzeClassVisitor.addImports(imports); + } + + @Override + public void visitMultiANewArrayInsn(String desc, int dims) { + addImportWithTypeDesc(desc); + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { + addImportWithInternalName(owner); + Arrays.asList(Type.getArgumentTypes(desc)).forEach(this::addImport); + addImport(Type.getReturnType(desc)); + } + + @Override + public void visitFieldInsn(int opcode, String owner, String name, String desc) { + addImportWithInternalName(owner); + addImportWithTypeDesc(desc); + } + + @Override + public void visitTypeInsn(int opcode, String type) { + addImportWithInternalName(type); + } + + @Override + public void visitTryCatchBlock(Label start, Label end, Label handler, String type) { + if (type != null) { //null means finally block + addImportWithInternalName(type); + } + } + + @Override + public void visitLocalVariable(String name, String desc, String signature, Label start, Label end, int index) { + addImportWithTypeDesc(desc); + } + + @Override + public void visitLdcInsn(Object constant) { + if (constant instanceof Type) { + addImport((Type) constant); + } + } + + @Override + public void visitInvokeDynamicInsn(String name, String desc, Handle bootstrapMethod, Object... bootstrapMethodArgs) { + for (Object arg : bootstrapMethodArgs) { + if (arg instanceof Type) { + addImport((Type) arg); + } else if (arg instanceof Handle) { + addImportWithInternalName(((Handle) arg).getOwner()); + Arrays.asList(Type.getArgumentTypes(desc)).forEach(this::addImport); + } else if ((arg instanceof Number) == false && (arg instanceof String) == false) { + throw new AssertionError("Unexpected type " + arg.getClass() + " with value '" + arg + "'"); + } + } + } + + @Override + public void visitMaxs(int maxStack, int maxLocals) { + } + + @Override + public void visitLineNumber(int line, Label start) { + } + + //only for debugging + @Override + public void visitLookupSwitchInsn(Label dflt, int[] keys, Label[] labels) { + } + + @Override + public void visitTableSwitchInsn(int min, int max, Label dflt, Label... labels) { + super.visitTableSwitchInsn(min, max, dflt, labels); + } + + @Override + public void visitIincInsn(int variable, int increment) { + } + + @Override + public void visitLabel(Label label) { + } + + @Override + public void visitJumpInsn(int opcode, Label label) { + } + + @Override + public void visitVarInsn(int opcode, int variable) { + } + + @Override + public void visitIntInsn(int opcode, int operand) { + } + + @Override + public void visitInsn(int opcode) { + } + + @Override + public void visitFrame(int type, int nLocal, Object[] local, int nStack, Object[] stack) { + } + + @Override + public void visitCode() { + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java new file mode 100644 index 00000000000..0f5fcf89f6a --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java @@ -0,0 +1,119 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.signature.SignatureReader; +import org.objectweb.asm.signature.SignatureVisitor; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author Tony Vaagenes + * @author ollivir + */ + +class AnalyzeSignatureVisitor extends SignatureVisitor implements ImportCollector { + private final AnalyzeClassVisitor analyzeClassVisitor; + private Set<String> imports = new HashSet<>(); + + AnalyzeSignatureVisitor(AnalyzeClassVisitor analyzeClassVisitor) { + super(Opcodes.ASM6); + this.analyzeClassVisitor = analyzeClassVisitor; + } + + public Set<String> imports() { + return imports; + } + + @Override + public void visitEnd() { + super.visitEnd(); + analyzeClassVisitor.addImports(imports); + } + + @Override + public void visitClassType(String className) { + addImportWithInternalName(className); + } + + @Override + public void visitFormalTypeParameter(String name) { + } + + @Override + public SignatureVisitor visitClassBound() { + return this; + } + + @Override + public SignatureVisitor visitInterfaceBound() { + return this; + } + + @Override + public SignatureVisitor visitSuperclass() { + return this; + } + + @Override + public SignatureVisitor visitInterface() { + return this; + } + + @Override + public SignatureVisitor visitParameterType() { + return this; + } + + @Override + public SignatureVisitor visitReturnType() { + return this; + } + + @Override + public SignatureVisitor visitExceptionType() { + return this; + } + + @Override + public void visitBaseType(char descriptor) { + } + + @Override + public void visitTypeVariable(String name) { + } + + @Override + public SignatureVisitor visitArrayType() { + return this; + } + + @Override + public void visitInnerClassType(String name) { + } + + @Override + public void visitTypeArgument() { + } + + @Override + public SignatureVisitor visitTypeArgument(char wildcard) { + return this; + } + + static void analyzeClass(String signature, AnalyzeClassVisitor analyzeClassVisitor) { + if (signature != null) { + new SignatureReader(signature).accept(new AnalyzeSignatureVisitor(analyzeClassVisitor)); + } + } + + static void analyzeMethod(String signature, AnalyzeClassVisitor analyzeClassVisitor) { + analyzeClass(signature, analyzeClassVisitor); + } + + static void analyzeField(String signature, AnalyzeClassVisitor analyzeClassVisitor) { + if (signature != null) + new SignatureReader(signature).acceptType(new AnalyzeSignatureVisitor(analyzeClassVisitor)); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.java new file mode 100644 index 00000000000..198618cabc4 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.java @@ -0,0 +1,35 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import java.util.Optional; +import java.util.Set; + +/** + * The result of analyzing a .class file. + * + * @author Tony Vaagenes + * @author ollivir + */ +public class ClassFileMetaData { + private final String name; + private final Set<String> referencedClasses; + private final Optional<ExportPackageAnnotation> exportPackage; + + public ClassFileMetaData(String name, Set<String> referencedClasses, Optional<ExportPackageAnnotation> exportPackage) { + this.name = name; + this.referencedClasses = referencedClasses; + this.exportPackage = exportPackage; + } + + public String getName() { + return name; + } + + public Set<String> getReferencedClasses() { + return referencedClasses; + } + + public Optional<ExportPackageAnnotation> getExportPackage() { + return exportPackage; + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.java new file mode 100644 index 00000000000..d2da9b5a226 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.java @@ -0,0 +1,62 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class ExportPackageAnnotation { + private final int major; + private final int minor; + private final int micro; + private final String qualifier; + + private static final Pattern QUALIFIER_PATTERN = Pattern.compile("[\\p{Alpha}\\p{Digit}_-]*"); + + public ExportPackageAnnotation(int major, int minor, int micro, String qualifier) { + this.major = major; + this.minor = minor; + this.micro = micro; + this.qualifier = qualifier; + + requireNonNegative(major, "major"); + requireNonNegative(minor, "minor"); + requireNonNegative(micro, "micro"); + if (QUALIFIER_PATTERN.matcher(qualifier).matches() == false) { + throw new IllegalArgumentException( + exportPackageError(String.format("qualifier must follow the format (alpha|digit|'_'|'-')* but was '%s'.", qualifier))); + } + } + + public String osgiVersion() { + return String.format("%d.%d.%d", major, minor, micro) + (qualifier.isEmpty() ? "" : "." + qualifier); + } + + private static String exportPackageError(String msg) { + return "ExportPackage anntotation: " + msg; + } + + private static void requireNonNegative(int i, String fieldName) { + if (i < 0) { + throw new IllegalArgumentException(exportPackageError(String.format("%s must be non-negative but was %d.", fieldName, i))); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ExportPackageAnnotation that = (ExportPackageAnnotation) o; + return major == that.major && minor == that.minor && micro == that.micro && Objects.equals(qualifier, that.qualifier); + } + + @Override + public int hashCode() { + return Objects.hash(major, minor, micro, qualifier); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ImportCollector.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ImportCollector.java new file mode 100644 index 00000000000..3946fe297f9 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ImportCollector.java @@ -0,0 +1,35 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import org.objectweb.asm.Type; + +import java.util.Collection; +import java.util.Optional; +import java.util.Set; + +/** + * @author ollivir + */ +public interface ImportCollector { + Set<String> imports(); + + default void addImportWithTypeDesc(String typeDescriptor) { + addImport(Type.getType(typeDescriptor)); + } + + default void addImport(Type type) { + addImport(Analyze.getClassName(type)); + } + + default void addImportWithInternalName(String name) { + addImport(Analyze.internalNameToClassName(name)); + } + + default void addImports(Collection<String> imports) { + imports().addAll(imports); + } + + default void addImport(Optional<String> anImport) { + anImport.ifPresent(pkg -> imports().add(pkg)); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/PackageTally.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/PackageTally.java new file mode 100644 index 00000000000..13bbc63192c --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/PackageTally.java @@ -0,0 +1,79 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import com.google.common.collect.Sets; +import com.yahoo.container.plugin.util.Maps; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class PackageTally { + private final Map<String, Optional<ExportPackageAnnotation>> definedPackagesMap; + private final Set<String> referencedPackagesUnfiltered; + + public PackageTally(Map<String, Optional<ExportPackageAnnotation>> definedPackagesMap, Set<String> referencedPackagesUnfiltered) { + this.definedPackagesMap = definedPackagesMap; + this.referencedPackagesUnfiltered = referencedPackagesUnfiltered; + } + + public Set<String> definedPackages() { + return definedPackagesMap.keySet(); + } + + public Set<String> referencedPackages() { + return Sets.difference(referencedPackagesUnfiltered, definedPackages()); + } + + public Map<String, ExportPackageAnnotation> exportedPackages() { + Map<String, ExportPackageAnnotation> ret = new HashMap<>(); + definedPackagesMap.forEach((k, v) -> { + v.ifPresent(annotation -> ret.put(k, annotation)); + }); + return ret; + } + + /** + * Represents the classes for two package tallies that are deployed as a single unit. + * <p> + * ExportPackageAnnotations from this has precedence over the other. + */ + public PackageTally combine(PackageTally other) { + Map<String, Optional<ExportPackageAnnotation>> map = Maps.combine(this.definedPackagesMap, other.definedPackagesMap, + (l, r) -> l.isPresent() ? l : r); + Set<String> referencedPkgs = new HashSet<>(this.referencedPackagesUnfiltered); + referencedPkgs.addAll(other.referencedPackagesUnfiltered); + + return new PackageTally(map, referencedPkgs); + } + + public static PackageTally combine(Collection<PackageTally> packageTallies) { + Map<String, Optional<ExportPackageAnnotation>> map = new HashMap<>(); + Set<String> referencedPkgs = new HashSet<>(); + + for (PackageTally pt : packageTallies) { + pt.definedPackagesMap.forEach((k, v) -> map.merge(k, v, (l, r) -> l.isPresent() ? l : r)); + referencedPkgs.addAll(pt.referencedPackagesUnfiltered); + } + return new PackageTally(map, referencedPkgs); + } + + public static PackageTally fromAnalyzedClassFiles(Collection<ClassFileMetaData> analyzedClassFiles) { + Map<String, Optional<ExportPackageAnnotation>> map = new HashMap<>(); + Set<String> referencedPkgs = new HashSet<>(); + + for (ClassFileMetaData metaData : analyzedClassFiles) { + String packageName = Packages.packageName(metaData.getName()); + map.merge(packageName, metaData.getExportPackage(), (l, r) -> l.isPresent() ? l : r); + metaData.getReferencedClasses().forEach(className -> referencedPkgs.add(Packages.packageName(className))); + } + return new PackageTally(map, referencedPkgs); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java new file mode 100644 index 00000000000..f9c6503c475 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java @@ -0,0 +1,43 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import java.util.HashSet; +import java.util.Set; + +/** + * Utility methods related to packages. + * + * @author Tony Vaagenes + * @author ollivir + */ +public class Packages { + public static class PackageMetaData { + public final Set<String> definedPackages; + public final Set<String> referencedExternalPackages; + + public PackageMetaData(Set<String> definedPackages, Set<String> referencedExternalPackages) { + this.definedPackages = definedPackages; + this.referencedExternalPackages = referencedExternalPackages; + } + } + + public static String packageName(String fullClassName) { + int index = fullClassName.lastIndexOf('.'); + if (index == -1) { + return ""; + } else { + return fullClassName.substring(0, index); + } + } + + public static PackageMetaData analyzePackages(Set<ClassFileMetaData> allClasses) { + Set<String> definedPackages = new HashSet<>(); + Set<String> referencedPackages = new HashSet<>(); + for (ClassFileMetaData metaData : allClasses) { + definedPackages.add(packageName(metaData.getName())); + metaData.getReferencedClasses().forEach(className -> referencedPackages.add(packageName(className))); + } + referencedPackages.removeAll(definedPackages); + return new PackageMetaData(definedPackages, referencedPackages); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/Artifacts.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/Artifacts.java new file mode 100644 index 00000000000..fff88d413d0 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/Artifacts.java @@ -0,0 +1,69 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.mojo; + +import org.apache.maven.artifact.Artifact; +import org.apache.maven.project.MavenProject; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class Artifacts { + public static class ArtifactSet { + private final List<Artifact> jarArtifactsToInclude; + private final List<Artifact> jarArtifactsProvided; + private final List<Artifact> nonJarArtifacts; + + private ArtifactSet(List<Artifact> jarArtifactsToInclude, List<Artifact> jarArtifactsProvided, List<Artifact> nonJarArtifacts) { + this.jarArtifactsToInclude = jarArtifactsToInclude; + this.jarArtifactsProvided = jarArtifactsProvided; + this.nonJarArtifacts = nonJarArtifacts; + } + + public List<Artifact> getJarArtifactsToInclude() { + return jarArtifactsToInclude; + } + + public List<Artifact> getJarArtifactsProvided() { + return jarArtifactsProvided; + } + + public List<Artifact> getNonJarArtifacts() { + return nonJarArtifacts; + } + } + + public static ArtifactSet getArtifacts(MavenProject project) { + + List<Artifact> jarArtifactsToInclude = new ArrayList<>(); + List<Artifact> jarArtifactsProvided = new ArrayList<>(); + List<Artifact> nonJarArtifactsToInclude = new ArrayList<>(); + List<Artifact> nonJarArtifactsProvided = new ArrayList<>(); + + for (Artifact artifact : project.getArtifacts()) { + if ("jar".equals(artifact.getType())) { + if (Artifact.SCOPE_COMPILE.equals(artifact.getScope())) { + jarArtifactsToInclude.add(artifact); + } else if (Artifact.SCOPE_PROVIDED.equals(artifact.getScope())) { + jarArtifactsProvided.add(artifact); + } + } else { + if (Artifact.SCOPE_COMPILE.equals(artifact.getScope())) { + nonJarArtifactsToInclude.add(artifact); + } else if (Artifact.SCOPE_PROVIDED.equals(artifact.getScope())) { + nonJarArtifactsProvided.add(artifact); + } + } + } + nonJarArtifactsToInclude.addAll(nonJarArtifactsProvided); + return new ArtifactSet(jarArtifactsToInclude, jarArtifactsProvided, nonJarArtifactsToInclude); + } + + public static Collection<Artifact> getArtifactsToInclude(MavenProject project) { + return getArtifacts(project).getJarArtifactsToInclude(); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.java new file mode 100644 index 00000000000..b5fac517c9d --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.java @@ -0,0 +1,145 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.mojo; + +import com.yahoo.container.plugin.util.Files; +import com.yahoo.container.plugin.util.JarFiles; +import org.apache.maven.archiver.MavenArchiveConfiguration; +import org.apache.maven.archiver.MavenArchiver; +import org.apache.maven.execution.MavenSession; +import org.apache.maven.model.Build; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.apache.maven.project.MavenProject; +import org.codehaus.plexus.archiver.jar.JarArchiver; + +import java.io.File; +import java.nio.channels.Channels; +import java.util.EnumMap; +import java.util.Enumeration; +import java.util.Map; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.zip.ZipEntry; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +@Mojo(name = "assemble-container-plugin", requiresDependencyResolution = ResolutionScope.COMPILE, threadSafe = true) +public class AssembleContainerPluginMojo extends AbstractMojo { + private static enum Dependencies { + WITH, WITHOUT + } + + @Parameter(defaultValue = "${project}") + private MavenProject project = null; + + @Parameter(defaultValue = "${session}", readonly = true, required = true) + private MavenSession session = null; + + @Parameter + private MavenArchiveConfiguration archiveConfiguration = new MavenArchiveConfiguration(); + + @Parameter(alias = "UseCommonAssemblyIds", defaultValue = "false") + private boolean useCommonAssemblyIds = false; + + @Override + public void execute() throws MojoExecutionException { + Map<Dependencies, String> jarSuffixes = new EnumMap<Dependencies, String>(Dependencies.class); + + if (useCommonAssemblyIds) { + jarSuffixes.put(Dependencies.WITHOUT, ".jar"); + jarSuffixes.put(Dependencies.WITH, "-jar-with-dependencies.jar"); + } else { + jarSuffixes.put(Dependencies.WITHOUT, "-without-dependencies.jar"); + jarSuffixes.put(Dependencies.WITH, "-deploy.jar"); + } + + Map<Dependencies, File> jarFiles = new EnumMap<Dependencies, File>(Dependencies.class); + jarSuffixes.forEach((dep, suffix) -> { + jarFiles.put(dep, jarFileInBuildDirectory(build().getFinalName(), suffix)); + }); + + // force recreating the archive + archiveConfiguration.setForced(true); + archiveConfiguration.setManifestFile(new File(new File(build().getOutputDirectory()), JarFile.MANIFEST_NAME)); + + JarArchiver jarWithoutDependencies = new JarArchiver(); + addClassesDirectory(jarWithoutDependencies); + createArchive(jarFiles.get(Dependencies.WITHOUT), jarWithoutDependencies); + project.getArtifact().setFile(jarFiles.get(Dependencies.WITHOUT)); + + JarArchiver jarWithDependencies = new JarArchiver(); + addClassesDirectory(jarWithDependencies); + addDependencies(jarWithDependencies); + createArchive(jarFiles.get(Dependencies.WITH), jarWithDependencies); + } + + private File jarFileInBuildDirectory(String name, String suffix) { + return new File(build().getDirectory(), name + suffix); + } + + private void addClassesDirectory(JarArchiver jarArchiver) { + File classesDirectory = new File(build().getOutputDirectory()); + if (classesDirectory.isDirectory()) { + jarArchiver.addDirectory(classesDirectory); + } + } + + private void createArchive(File jarFile, JarArchiver jarArchiver) throws MojoExecutionException { + MavenArchiver mavenArchiver = new MavenArchiver(); + mavenArchiver.setArchiver(jarArchiver); + mavenArchiver.setOutputFile(jarFile); + try { + mavenArchiver.createArchive(session, project, archiveConfiguration); + } catch (Exception e) { + throw new MojoExecutionException("Error creating archive " + jarFile.getName(), e); + } + } + + private void addDependencies(JarArchiver jarArchiver) { + Artifacts.getArtifactsToInclude(project).forEach(artifact -> { + if ("jar".equals(artifact.getType())) { + jarArchiver.addFile(artifact.getFile(), "dependencies/" + artifact.getFile().getName()); + copyConfigDefinitions(artifact.getFile(), jarArchiver); + } else { + getLog().warn("Unkown artifact type " + artifact.getType()); + } + }); + } + + private void copyConfigDefinitions(File file, JarArchiver jarArchiver) { + JarFiles.withJarFile(file, jarFile -> { + for (Enumeration<JarEntry> entries = jarFile.entries(); entries.hasMoreElements();) { + JarEntry entry = entries.nextElement(); + String name = entry.getName(); + if (name.startsWith("configdefinitions/") && name.endsWith(".def")) { + copyConfigDefinition(jarFile, entry, jarArchiver); + } + } + return null; + }); + } + + private void copyConfigDefinition(JarFile jarFile, ZipEntry entry, JarArchiver jarArchiver) { + JarFiles.withInputStream(jarFile, entry, input -> { + String defPath = entry.getName().replace("/", File.separator); + File destinationFile = new File(build().getOutputDirectory(), defPath); + destinationFile.getParentFile().mkdirs(); + + Files.withFileOutputStream(destinationFile, output -> { + output.getChannel().transferFrom(Channels.newChannel(input), 0, Long.MAX_VALUE); + return null; + }); + jarArchiver.addFile(destinationFile, entry.getName()); + return null; + }); + } + + private Build build() { + return project.getBuild(); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java new file mode 100644 index 00000000000..b2abf13695f --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java @@ -0,0 +1,115 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.mojo; + +import com.google.common.base.Preconditions; +import com.yahoo.container.plugin.bundle.AnalyzeBundle; +import com.yahoo.container.plugin.osgi.ProjectBundleClassPaths; +import com.yahoo.container.plugin.osgi.ProjectBundleClassPaths.BundleClasspathMapping; +import org.apache.maven.artifact.Artifact; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.apache.maven.project.MavenProject; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Generates mapping from Bundle-SymbolicName to classpath elements, e.g myBundle -> [.m2/repository/com/mylib/Mylib.jar, + * myBundleProject/target/classes] The mapping in stored in a json file. + * + * @author Tony Vaagenes + * @author ollivir + */ +@Mojo(name = "generate-bundle-classpath-mappings", requiresDependencyResolution = ResolutionScope.COMPILE_PLUS_RUNTIME, threadSafe = true) +public class GenerateBundleClassPathMappingsMojo extends AbstractMojo { + @Parameter(defaultValue = "${project}") + private MavenProject project = null; + + //TODO: Combine with com.yahoo.container.plugin.mojo.GenerateOsgiManifestMojo.bundleSymbolicName + @Parameter(alias = "Bundle-SymbolicName", defaultValue = "${project.artifactId}") + private String bundleSymbolicName = null; + + /* Sample output -- target/test-classes/bundle-plugin.bundle-classpath-mappings.json + { + "mainBundle": { + "bundleSymbolicName": "bundle-plugin-test", + "classPathElements": [ + "/Users/tonyv/Repos/vespa/bundle-plugin-test/target/classes", + "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/6-SNAPSHOT/jrt-6-SNAPSHOT.jar", + "/Users/tonyv/.m2/repository/com/yahoo/vespa/annotations/6-SNAPSHOT/annotations-6-SNAPSHOT.jar" + ] + }, + "providedDependencies": [ + { + "bundleSymbolicName": "jrt", + "classPathElements": [ + "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/6-SNAPSHOT/jrt-6-SNAPSHOT.jar" + ] + } + ] + } + */ + @Override + public void execute() throws MojoExecutionException { + Preconditions.checkNotNull(bundleSymbolicName); + + Artifacts.ArtifactSet artifacts = Artifacts.getArtifacts(project); + List<Artifact> embeddedArtifacts = artifacts.getJarArtifactsToInclude(); + List<Artifact> providedJarArtifacts = artifacts.getJarArtifactsProvided(); + + List<File> embeddedArtifactsFiles = embeddedArtifacts.stream().map(Artifact::getFile).collect(Collectors.toList()); + + List<String> classPathElements = Stream.concat(Stream.of(outputDirectory()), embeddedArtifactsFiles.stream()) + .map(File::getAbsolutePath).collect(Collectors.toList()); + + ProjectBundleClassPaths classPathMappings = new ProjectBundleClassPaths( + new BundleClasspathMapping(bundleSymbolicName, classPathElements), + providedJarArtifacts.stream().map(f -> createDependencyClasspathMapping(f)).filter(Optional::isPresent).map(Optional::get) + .collect(Collectors.toList())); + + try { + ProjectBundleClassPaths.save(testOutputPath().resolve(ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME), classPathMappings); + } catch (IOException e) { + throw new MojoExecutionException("Error saving to file " + testOutputPath(), e); + } + } + + private File outputDirectory() { + return new File(project.getBuild().getOutputDirectory()); + } + + private Path testOutputPath() { + return Paths.get(project.getBuild().getTestOutputDirectory()); + } + + /* TODO: + * 1) add the dependencies of the artifact in the future(i.e. dependencies of dependencies) + * or + * 2) obtain bundles with embedded dependencies from the maven repository, + * and support loading classes from the nested jar files in those bundles. + */ + Optional<BundleClasspathMapping> createDependencyClasspathMapping(Artifact artifact) { + return bundleSymbolicNameForArtifact(artifact) + .map(name -> new BundleClasspathMapping(name, Arrays.asList(artifact.getFile().getAbsolutePath()))); + } + + private static Optional<String> bundleSymbolicNameForArtifact(Artifact artifact) { + if (artifact.getFile().getName().endsWith(".jar")) { + return AnalyzeBundle.bundleSymbolicName(artifact.getFile()); + } else { + // Not the best heuristic. The other alternatives are parsing the pom file or + // storing information in target/classes when building the provided bundles. + return Optional.of(artifact.getArtifactId()); + } + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java new file mode 100644 index 00000000000..8d19f112765 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java @@ -0,0 +1,313 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.mojo; + +import com.google.common.collect.Sets; +import com.yahoo.container.plugin.bundle.AnalyzeBundle; +import com.yahoo.container.plugin.classanalysis.Analyze; +import com.yahoo.container.plugin.classanalysis.ClassFileMetaData; +import com.yahoo.container.plugin.classanalysis.ExportPackageAnnotation; +import com.yahoo.container.plugin.classanalysis.PackageTally; +import com.yahoo.container.plugin.osgi.ExportPackageParser; +import com.yahoo.container.plugin.osgi.ExportPackages; +import com.yahoo.container.plugin.osgi.ExportPackages.Export; +import com.yahoo.container.plugin.osgi.ImportPackages; +import com.yahoo.container.plugin.osgi.ImportPackages.Import; +import com.yahoo.container.plugin.util.Strings; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.maven.artifact.Artifact; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.apache.maven.project.MavenProject; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.container.plugin.util.Files.allDescendantFiles; +import static com.yahoo.container.plugin.util.IO.withFileOutputStream; +import static com.yahoo.container.plugin.util.JarFiles.withInputStream; +import static com.yahoo.container.plugin.util.JarFiles.withJarFile; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +@Mojo(name = "generate-osgi-manifest", requiresDependencyResolution = ResolutionScope.TEST, threadSafe = true) +public class GenerateOsgiManifestMojo extends AbstractMojo { + + @Parameter(defaultValue = "${project}") + private MavenProject project = null; + + @Parameter + private String discApplicationClass = null; + + @Parameter + private String discPreInstallBundle = null; + + @Parameter(alias = "Bundle-Version", defaultValue = "${project.version}") + private String bundleVersion = null; + + @Parameter(alias = "Bundle-SymbolicName", defaultValue = "${project.artifactId}") + private String bundleSymbolicName = null; + + @Parameter(alias = "Bundle-Activator") + private String bundleActivator = null; + + @Parameter(alias = "X-JDisc-Privileged-Activator") + private String jdiscPrivilegedActivator = null; + + @Parameter(alias = "X-Config-Models") + private String configModels = null; + + @Parameter(alias = "Import-Package") + private String importPackage = null; + + @Parameter(alias = "WebInfUrl") + private String webInfUrl = null; + + @Parameter(alias = "Main-Class") + private String mainClass = null; + + @Parameter(alias = "X-Jersey-Binding") + private String jerseyBinding = null; + + public void execute() throws MojoExecutionException, MojoFailureException { + try { + Artifacts.ArtifactSet artifactSet = Artifacts.getArtifacts(project); + warnOnUnsupportedArtifacts(artifactSet.getNonJarArtifacts()); + + AnalyzeBundle.PublicPackages publicPackagesFromProvidedJars = AnalyzeBundle.publicPackagesAggregated( + artifactSet.getJarArtifactsProvided().stream().map(Artifact::getFile).collect(Collectors.toList())); + PackageTally includedJarPackageTally = definedPackages(artifactSet.getJarArtifactsToInclude()); + + PackageTally projectPackageTally = analyzeProjectClasses(); + PackageTally pluginPackageTally = projectPackageTally.combine(includedJarPackageTally); + + Set<String> definedPackages = new HashSet<>(projectPackageTally.definedPackages()); + definedPackages.addAll(includedJarPackageTally.definedPackages()); + + warnIfPackagesDefinedOverlapsGlobalPackages(definedPackages, publicPackagesFromProvidedJars.globals); + + if (getLog().isDebugEnabled()) { + getLog().debug("Referenced packages = " + pluginPackageTally.referencedPackages()); + getLog().debug("Defined packages = " + pluginPackageTally.definedPackages()); + getLog().debug("Exported packages of dependencies = " + publicPackagesFromProvidedJars.exports.stream() + .map(e -> "(" + e.getPackageNames().toString() + ", " + e.version().orElse("")).collect(Collectors.joining(", "))); + } + + Map<String, Import> calculatedImports = ImportPackages.calculateImports(pluginPackageTally.referencedPackages(), + pluginPackageTally.definedPackages(), ExportPackages.exportsByPackageName(publicPackagesFromProvidedJars.exports)); + + Map<String, Optional<String>> manualImports = emptyToNone(importPackage).map(GenerateOsgiManifestMojo::getManualImports) + .orElseGet(HashMap::new); + for (String packageName : manualImports.keySet()) { + calculatedImports.remove(packageName); + } + createManifestFile(new File(project.getBuild().getOutputDirectory()), manifestContent(project, + artifactSet.getJarArtifactsToInclude(), manualImports, calculatedImports.values(), pluginPackageTally)); + + } catch (Exception e) { + throw new MojoExecutionException("Failed generating osgi manifest.", e); + } + } + + private static void warnIfPackagesDefinedOverlapsGlobalPackages(Set<String> internalPackages, List<String> globalPackages) + throws MojoExecutionException { + Set<String> overlap = Sets.intersection(internalPackages, new HashSet<>(globalPackages)); + if (overlap.isEmpty() == false) { + throw new MojoExecutionException( + "The following packages are both global and included in the bundle:\n " + String.join("\n ", overlap)); + } + } + + private Collection<String> osgiExportPackages(Map<String, ExportPackageAnnotation> exportedPackages) { + return exportedPackages.entrySet().stream().map(entry -> entry.getKey() + ";version=" + entry.getValue().osgiVersion()) + .collect(Collectors.toList()); + } + + private static String trimWhitespace(Optional<String> lines) { + return Stream.of(lines.orElse("").split(",")).map(String::trim).collect(Collectors.joining(",")); + } + + private Map<String, String> manifestContent(MavenProject project, Collection<Artifact> jarArtifactsToInclude, + Map<String, Optional<String>> manualImports, Collection<Import> imports, PackageTally pluginPackageTally) { + Map<String, String> ret = new HashMap<>(); + String importPackage = Stream.concat(manualImports.entrySet().stream().map(e -> asOsgiImport(e.getKey(), e.getValue())), + imports.stream().map(Import::asOsgiImport)).sorted().collect(Collectors.joining(",")); + String exportPackage = osgiExportPackages(pluginPackageTally.exportedPackages()).stream().sorted().collect(Collectors.joining(",")); + + for (Pair<String, String> element : Arrays.asList(// + Pair.of("Created-By", "vespa container maven plugin"), // + Pair.of("Bundle-ManifestVersion", "2"), // + Pair.of("Bundle-Name", project.getName()), // + Pair.of("Bundle-SymbolicName", bundleSymbolicName), // + Pair.of("Bundle-Version", asBundleVersion(bundleVersion)), // + Pair.of("Bundle-Vendor", "Yahoo!"), // + Pair.of("Bundle-ClassPath", bundleClassPath(jarArtifactsToInclude)), // + Pair.of("Bundle-Activator", bundleActivator), // + Pair.of("X-JDisc-Privileged-Activator", jdiscPrivilegedActivator), // + Pair.of("Main-Class", mainClass), // + Pair.of("X-JDisc-Application", discApplicationClass), // + Pair.of("X-JDisc-Preinstall-Bundle", trimWhitespace(Optional.ofNullable(discPreInstallBundle))), // + Pair.of("X-Config-Models", configModels), // + Pair.of("X-Jersey-Binding", jerseyBinding), // + Pair.of("WebInfUrl", webInfUrl), // + Pair.of("Import-Package", importPackage), // + Pair.of("Export-Package", exportPackage))) { + if (element.getValue() != null && element.getValue().isEmpty() == false) { + ret.put(element.getKey(), element.getValue()); + } + } + return ret; + } + + private static String asOsgiImport(String packageName, Optional<String> version) { + return version.map(s -> packageName + ";version=" + quote(s)).orElse(packageName); + } + + private static String quote(String s) { + return "\"" + s + "\""; + } + + private static void createManifestFile(File outputDirectory, Map<String, String> manifestContent) { + Manifest manifest = toManifest(manifestContent); + + withFileOutputStream(new File(outputDirectory, JarFile.MANIFEST_NAME), outputStream -> { + manifest.write(outputStream); + return null; + }); + } + + private static Manifest toManifest(Map<String, String> manifestContent) { + Manifest manifest = new Manifest(); + Attributes mainAttributes = manifest.getMainAttributes(); + + mainAttributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifestContent.forEach(mainAttributes::putValue); + + return manifest; + } + + private static String bundleClassPath(Collection<Artifact> artifactsToInclude) { + return Stream.concat(Stream.of("."), artifactsToInclude.stream().map(GenerateOsgiManifestMojo::dependencyPath)) + .collect(Collectors.joining(",")); + } + + private static String dependencyPath(Artifact artifact) { + return "dependencies/" + artifact.getFile().getName(); + } + + private static String asBundleVersion(String projectVersion) { + if (projectVersion == null) { + throw new IllegalArgumentException("Missing project version."); + } + + String[] parts = projectVersion.split("-", 2); + List<String> numericPart = Stream.of(parts[0].split("\\.")).map(s -> Strings.replaceEmptyString(s, "0")).limit(3) + .collect(Collectors.toList()); + while (numericPart.size() < 3) { + numericPart.add("0"); + } + + return String.join(".", numericPart); + } + + private void warnOnUnsupportedArtifacts(Collection<Artifact> nonJarArtifacts) { + List<Artifact> unsupportedArtifacts = nonJarArtifacts.stream().filter(a -> "pom".equals(a.getType()) == false) + .collect(Collectors.toList()); + + unsupportedArtifacts.forEach(artifact -> getLog() + .warn(String.format("Unsupported artifact '%s': Type '%s' is not supported. Please file a feature request.", + artifact.getId(), artifact.getType()))); + } + + private PackageTally analyzeProjectClasses() { + File outputDirectory = new File(project.getBuild().getOutputDirectory()); + + List<ClassFileMetaData> analyzedClasses = allDescendantFiles(outputDirectory).filter(file -> file.getName().endsWith(".class")) + .map(Analyze::analyzeClass).collect(Collectors.toList()); + + return PackageTally.fromAnalyzedClassFiles(analyzedClasses); + } + + private static PackageTally definedPackages(Collection<Artifact> jarArtifacts) { + return PackageTally.combine(jarArtifacts.stream().map(ja -> withJarFile(ja.getFile(), GenerateOsgiManifestMojo::definedPackages)) + .collect(Collectors.toList())); + } + + private static PackageTally definedPackages(JarFile jarFile) throws MojoExecutionException { + List<ClassFileMetaData> analyzedClasses = new ArrayList<>(); + for (Enumeration<JarEntry> entries = jarFile.entries(); entries.hasMoreElements();) { + JarEntry entry = entries.nextElement(); + if (entry.isDirectory() == false && entry.getName().endsWith(".class")) { + analyzedClasses.add(analyzeClass(jarFile, entry)); + } + } + return PackageTally.fromAnalyzedClassFiles(analyzedClasses); + } + + private static ClassFileMetaData analyzeClass(JarFile jarFile, JarEntry entry) throws MojoExecutionException { + try { + return withInputStream(jarFile, entry, Analyze::analyzeClass); + } catch (Exception e) { + throw new MojoExecutionException( + String.format("While analyzing the class '%s' in jar file '%s'", entry.getName(), jarFile.getName()), e); + } + } + + private static Map<String, Optional<String>> getManualImports(String importPackage) { + try { + Map<String, Optional<String>> ret = new HashMap<>(); + List<Export> imports = parseImportPackages(importPackage); + for (Export imp : imports) { + Optional<String> version = getVersionThrowOthers(imp.getParameters()); + imp.getPackageNames().forEach(pn -> ret.put(pn, version)); + } + + return ret; + } catch (Exception e) { + throw new RuntimeException("Error in Import-Package:" + importPackage, e); + } + } + + private static Optional<String> getVersionThrowOthers(List<ExportPackages.Parameter> parameters) { + if (parameters.size() == 1 && "version".equals(parameters.get(0).getName())) { + return Optional.of(parameters.get(0).getValue()); + } else if (parameters.size() == 0) { + return Optional.empty(); + } else { + List<String> paramNames = parameters.stream().map(ExportPackages.Parameter::getName).collect(Collectors.toList()); + throw new RuntimeException("A single, optional version parameter expected, but got " + paramNames); + } + } + + private static List<Export> parseImportPackages(String importPackages) { + return ExportPackageParser.parseExports(importPackages); + } + + private static Optional<String> emptyToNone(String str) { + return Optional.ofNullable(str).map(String::trim).filter(s -> s.isEmpty() == false); + } + + private static boolean isClassToAnalyze(String name) { + return name.endsWith(".class") && name.endsWith("module-info.class") == false; + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackageParser.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackageParser.java new file mode 100644 index 00000000000..16858808a58 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackageParser.java @@ -0,0 +1,283 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.osgi; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class ExportPackageParser { + public static List<ExportPackages.Export> parseExports(String exportAttribute) { + ParsingContext p = new ParsingContext(exportAttribute.trim()); + + List<ExportPackages.Export> exports = parseExportPackage(p); + if (exports.isEmpty()) { + p.fail("Expected a list of exports"); + } else if (p.atEnd() == false) { + p.fail("Exports not fully processed"); + } + return exports; + } + + private static class ParsingContext { + private enum State { + Invalid, WantMore, End + } + + private CharSequence input; + private int pos; + private State state; + private int length; + private char ch; + + private ParsingContext(CharSequence input) { + this.input = input; + this.pos = 0; + } + + private Optional<String> read(Consumer<ParsingContext> rule) { + StringBuilder ret = new StringBuilder(); + + parse: while (true) { + if (input.length() < pos + 1) { + break; + } + ch = input.charAt(pos); + state = State.WantMore; + length = ret.length(); + rule.accept(this); + + switch (state) { + case Invalid: + if (ret.length() == 0) { + break parse; + } else { + String printable = Character.isISOControl(ch) ? "#" + Integer.toString((int) ch) + : "[" + Character.toString(ch) + "]"; + pos++; + fail("Character " + printable + " was not acceptable"); + } + break; + case WantMore: + ret.append(ch); + pos++; + break; + case End: + break parse; + } + } + + if (ret.length() == 0) { + return Optional.empty(); + } else { + return Optional.of(ret.toString()); + } + } + + private Optional<String> regexp(Pattern pattern) { + Matcher matcher = pattern.matcher(input); + matcher.region(pos, input.length()); + if (matcher.lookingAt()) { + String value = matcher.group(); + pos += value.length(); + return Optional.of(value); + } else { + return Optional.empty(); + } + } + + private Optional<String> exactly(String string) { + if (input.length() - pos < string.length()) { + return Optional.empty(); + } + if (input.subSequence(pos, pos + string.length()).equals(string)) { + pos += string.length(); + return Optional.of(string); + } + return Optional.empty(); + } + + private boolean atEnd() { + return pos == input.length(); + } + + private void invalid() { + this.state = State.Invalid; + } + + private void end() { + this.state = State.End; + } + + private void fail(String message) { + throw new RuntimeException("Failed parsing Export-Package: " + message + " at position " + pos); + } + } + + /* ident = ? a valid Java identifier ? */ + private static Optional<String> parseIdent(ParsingContext p) { + Optional<String> ident = p.read(ctx -> { + if (ctx.length == 0) { + if (Character.isJavaIdentifierStart(ctx.ch) == false) { + ctx.invalid(); + } + } else { + if (Character.isJavaIdentifierPart(ctx.ch) == false) { + ctx.end(); + } + } + }); + return ident; + } + + /* stringLiteral = ? sequence of any character except double quotes, control characters or backslash, + a backslash followed by another backslash, a single or double quote, or one of the letters b,f,n,r or t + a backslash followed by u followed by four hexadecimal digits ? */ + private static Pattern STRING_LITERAL_PATTERN = Pattern + .compile("\"" + "(?:[^\"\\p{Cntrl}\\\\]|\\\\[\\\\'\"bfnrt]|\\\\u[0-9a-fA-F]{4})+" + "\""); + + private static Optional<String> parseStringLiteral(ParsingContext p) { + return p.regexp(STRING_LITERAL_PATTERN).map(quoted -> quoted.substring(1, quoted.length() - 1)); + } + + /* extended = { \p{Alnum} | '_' | '-' | '.' }+ */ + private static Pattern EXTENDED_PATTERN = Pattern.compile("[\\p{Alnum}_.-]+"); + + private static Optional<String> parseExtended(ParsingContext p) { + return p.regexp(EXTENDED_PATTERN); + } + + /* argument = extended | stringLiteral | ? failure ? */ + private static String parseArgument(ParsingContext p) { + Optional<String> argument = parseExtended(p); + if (argument.isPresent() == false) { + argument = parseStringLiteral(p); + } + if (argument.isPresent() == false) { + p.fail("Expected an extended token or a string literal"); + } + return argument.get(); + } + + /* + * parameter = ( directive | attribute ) + * directive = extended, ':=', argument + * attribute = extended, '=', argument + */ + private static Pattern DIRECTIVE_OR_ATTRIBUTE_SEPARATOR_PATTERN = Pattern.compile("\\s*:?=\\s*"); + + private static Optional<ExportPackages.Parameter> parseParameter(ParsingContext p) { + int backtrack = p.pos; + Optional<String> ext = parseExtended(p); + if (ext.isPresent()) { + Optional<String> sep = p.regexp(DIRECTIVE_OR_ATTRIBUTE_SEPARATOR_PATTERN); + if (sep.isPresent() == false) { + p.pos = backtrack; + return Optional.empty(); + } + String argument = parseArgument(p); + return Optional.of(new ExportPackages.Parameter(ext.get(), argument)); + } else { + return Optional.empty(); + } + } + + /* parameters = parameter, { ';' parameter } */ + private static Pattern PARAMETER_SEPARATOR_PATTERN = Pattern.compile("\\s*;\\s*"); + + private static List<ExportPackages.Parameter> parseParameters(ParsingContext p) { + List<ExportPackages.Parameter> params = new ArrayList<>(); + boolean wantMore = true; + do { + Optional<ExportPackages.Parameter> param = parseParameter(p); + if (param.isPresent()) { + params.add(param.get()); + wantMore = p.regexp(PARAMETER_SEPARATOR_PATTERN).isPresent(); + } else { + wantMore = false; + } + } while (wantMore); + + return params; + } + + /* packageName = ident, { '.', ident } */ + private static Optional<String> parsePackageName(ParsingContext p) { + StringBuilder ret = new StringBuilder(); + + boolean wantMore = true; + do { + Optional<String> ident = parseIdent(p); + if (ident.isPresent()) { + ret.append(ident.get()); + Optional<String> separator = p.exactly("."); + if (separator.isPresent()) { + ret.append(separator.get()); + wantMore = true; + } else { + wantMore = false; + } + } else { + wantMore = false; + } + } while (wantMore); + + if (ret.length() > 0) { + return Optional.of(ret.toString()); + } else { + return Optional.empty(); + } + } + + /* export = packageName, [ ';', ( parameters | export ) ] */ + private static ExportPackages.Export parseExport(ParsingContext p) { + List<String> exports = new ArrayList<>(); + + boolean wantMore = true; + do { + if (exports.isEmpty() == false) { // second+ iteration + List<ExportPackages.Parameter> params = parseParameters(p); + if (params.isEmpty() == false) { + return new ExportPackages.Export(exports, params); + } + } + + Optional<String> packageName = parsePackageName(p); + if (packageName.isPresent()) { + exports.add(packageName.get()); + } else { + p.fail(exports.isEmpty() ? "Expected a package name" : "Expected either a package name or a parameter list"); + } + + wantMore = p.regexp(PARAMETER_SEPARATOR_PATTERN).isPresent(); + } while (wantMore); + + return new ExportPackages.Export(exports, new ArrayList<>()); + } + + /* exportPackage = export, { ',', export } */ + private static Pattern EXPORT_SEPARATOR_PATTERN = Pattern.compile("\\s*,\\s*"); + + private static List<ExportPackages.Export> parseExportPackage(ParsingContext p) { + List<ExportPackages.Export> exports = new ArrayList<>(); + + boolean wantMore = true; + do { + ExportPackages.Export export = parseExport(p); + if (export.getPackageNames().isEmpty()) { + wantMore = false; + } else { + exports.add(export); + wantMore = p.regexp(EXPORT_SEPARATOR_PATTERN).isPresent(); + } + } while (wantMore); + + return exports; + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackages.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackages.java new file mode 100644 index 00000000000..253e0727050 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackages.java @@ -0,0 +1,70 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.osgi; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class ExportPackages { + public static class Export { + private final List<String> packageNames; + private final List<Parameter> parameters; + + public Export(List<String> packageNames, List<Parameter> parameters) { + this.packageNames = packageNames; + this.parameters = parameters; + } + + public Optional<String> version() { + for (Parameter par : parameters) { + if ("version".equals(par.getName())) { + return Optional.of(par.getValue()); + } + } + return Optional.empty(); + } + + public List<String> getPackageNames() { + return packageNames; + } + + public List<Parameter> getParameters() { + return parameters; + } + } + + public static class Parameter { + private final String name; + private final String value; + + public Parameter(String name, String value) { + this.name = name; + this.value = value; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + } + + public static Map<String, Export> exportsByPackageName(Collection<Export> exports) { + Map<String, Export> ret = new HashMap<>(); + for (Export export : exports) { + for (String packageName : export.getPackageNames()) { + //ensure that earlier exports of a package overrides later exports. + ret.computeIfAbsent(packageName, ign -> export); + } + } + return ret; + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ImportPackages.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ImportPackages.java new file mode 100644 index 00000000000..b58248ec4a6 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ImportPackages.java @@ -0,0 +1,97 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.osgi; + +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class ImportPackages { + public static final int INFINITE_VERSION = 99999; + private static final String GUAVA_BASE_PACKAGE = "com.google.common"; + + public static class Import { + private final String packageName; + private final List<Integer> versionNumber; + + public Import(String packageName, Optional<String> version) { + this.packageName = packageName; + this.versionNumber = new ArrayList<>(); + + if (version.isPresent()) { + try { + Arrays.stream(version.get().split("\\.")).map(Integer::parseInt).limit(3).forEach(this.versionNumber::add); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + String.format("Invalid version number '%s' for package '%s'.", version.get(), packageName), e); + } + } + } + + public Optional<Integer> majorVersion() { + if (versionNumber.size() >= 1) { + return Optional.of(versionNumber.get(0)); + } else { + return Optional.empty(); + } + } + + public String packageName() { + return packageName; + } + + public String version() { + return versionNumber.stream().map(Object::toString).collect(Collectors.joining(".")); + } + + // TODO: Detecting guava packages should be based on Bundle-SymbolicName, not package name. + public Optional<String> importVersionRange() { + if (versionNumber.isEmpty()) { + return Optional.empty(); + } else { + int upperLimit = isGuavaPackage() ? INFINITE_VERSION // guava increases major version for each release + : versionNumber.get(0) + 1; + return Optional.of(String.format("[%s,%d)", version(), upperLimit)); + } + } + + public boolean isGuavaPackage() { + return packageName.equals(GUAVA_BASE_PACKAGE) || packageName.startsWith(GUAVA_BASE_PACKAGE + "."); + } + + public String asOsgiImport() { + return packageName + importVersionRange().map(version -> ";version=\"" + version + '"').orElse(""); + } + } + + public static Map<String, Import> calculateImports(Set<String> referencedPackages, Set<String> implementedPackages, + Map<String, ExportPackages.Export> exportedPackages) { + Map<String, Import> ret = new HashMap<>(); + for (String undefinedPackage : Sets.difference(referencedPackages, implementedPackages)) { + ExportPackages.Export export = exportedPackages.get(undefinedPackage); + if (export != null) { + ret.put(undefinedPackage, new Import(undefinedPackage, version(export))); + } + } + return ret; + } + + private static Optional<String> version(ExportPackages.Export export) { + for (ExportPackages.Parameter param : export.getParameters()) { + if ("version".equals(param.getName())) { + return Optional.of(param.getValue()); + } + } + return Optional.empty(); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/Files.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/Files.java new file mode 100644 index 00000000000..bcd5d3768f3 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/Files.java @@ -0,0 +1,30 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.util; + +import java.io.File; +import java.io.FileOutputStream; +import java.util.stream.Stream; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class Files { + public static Stream<File> allDescendantFiles(File file) { + if (file.isFile()) { + return Stream.of(file); + } else if (file.isDirectory()) { + return Stream.of(file.listFiles()).flatMap(Files::allDescendantFiles); + } else { + return Stream.empty(); + } + } + + public static <T> T withFileOutputStream(File file, ThrowingFunction<FileOutputStream, T> f) { + try (FileOutputStream fos = new FileOutputStream(file)) { + return f.apply(fos); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/IO.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/IO.java new file mode 100644 index 00000000000..a1e313b920b --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/IO.java @@ -0,0 +1,41 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.util; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.OutputStream; + +/** + * Utility methods relating to IO + * + * @author Tony Vaagenes + * @author ollivir + */ +public class IO { + public static <T> T withFileInputStream(File file, ThrowingFunction<FileInputStream, T> f) { + try (FileInputStream fis = new FileInputStream(file)) { + return f.apply(fis); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Creates a new file and all its parent directories, and provides a file output stream to the file. + */ + public static <T> T withFileOutputStream(File file, ThrowingFunction<OutputStream, T> f) { + makeDirectoriesRecursive(file.getParentFile()); + try (FileOutputStream fos = new FileOutputStream(file)) { + return f.apply(fos); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static void makeDirectoriesRecursive(File file) { + if (!file.mkdirs() && !file.isDirectory()) { + throw new RuntimeException("Could not create directory " + file.getPath()); + } + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/JarFiles.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/JarFiles.java new file mode 100644 index 00000000000..398b2f5a72a --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/JarFiles.java @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.util; + +import java.io.File; +import java.io.InputStream; +import java.util.Optional; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class JarFiles { + public static <T> T withJarFile(File file, ThrowingFunction<JarFile, T> action) { + try (JarFile jar = new JarFile(file)) { + return action.apply(jar); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static <T> T withInputStream(ZipFile zipFile, ZipEntry zipEntry, ThrowingFunction<InputStream, T> action) { + try (InputStream is = zipFile.getInputStream(zipEntry)) { + return action.apply(is); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static Optional<Manifest> getManifest(File jarFile) { + return withJarFile(jarFile, jar -> Optional.ofNullable(jar.getManifest())); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/Maps.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/Maps.java new file mode 100644 index 00000000000..5aa14402d4e --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/Maps.java @@ -0,0 +1,31 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.util; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.BiFunction; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class Maps { + public static <K, V> Map<K, V> combine(Map<K, V> left, Map<K, V> right, BiFunction<V, V, V> combiner) { + Map<K, V> ret = new HashMap<>(); + Set<K> keysRight = new HashSet<>(right.keySet()); + + left.forEach((k, v) -> { + if (keysRight.contains(k)) { + ret.put(k, combiner.apply(v, right.get(k))); + keysRight.remove(k); + } else { + ret.put(k, v); + } + }); + keysRight.forEach(k -> ret.put(k, right.get(k))); + + return ret; + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/Strings.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/Strings.java new file mode 100644 index 00000000000..15bdfb153ad --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/Strings.java @@ -0,0 +1,26 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.util; + +import java.util.Optional; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class Strings { + public static String replaceEmptyString(String s, String replacement) { + if (s == null || s.isEmpty()) { + return replacement; + } else { + return s; + } + } + + public static Optional<String> noneIfEmpty(String s) { + if (s == null || s.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(s); + } + } +}
\ No newline at end of file diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/ThrowingFunction.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/ThrowingFunction.java new file mode 100644 index 00000000000..9ca64aabd73 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/ThrowingFunction.java @@ -0,0 +1,11 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.util; + +/* Equivalent to java.util.function.Function, but allows throwing of Exceptions */ + +/** + * @author ollivir + */ +public interface ThrowingFunction<T, U> { + U apply(T input) throws Exception; +} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/bundle/AnalyzeBundle.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/bundle/AnalyzeBundle.scala deleted file mode 100644 index 1b3979476bd..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/bundle/AnalyzeBundle.scala +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.bundle - -import java.io.File -import java.util.jar.{Manifest => JarManifest} - -import com.yahoo.container.plugin.osgi.ExportPackageParser -import com.yahoo.container.plugin.osgi.ExportPackages.Export -import com.yahoo.container.plugin.util.JarFiles - - -/** - * @author tonytv - */ -object AnalyzeBundle { - case class PublicPackages(exports : List[Export], globals : List[String]) - - def publicPackagesAggregated(jarFiles : Iterable[File]) = aggregate(jarFiles map {publicPackages(_)}) - - def aggregate(publicPackagesList : Iterable[PublicPackages]) = - (PublicPackages(List(), List()) /: publicPackagesList) { (a,b) => - PublicPackages(a.exports ++ b.exports, a.globals ++ b.globals) - } - - def publicPackages(jarFile: File): PublicPackages = { - try { - - (for { - manifest <- JarFiles.getManifest(jarFile) - if isOsgiManifest(manifest) - } yield PublicPackages(parseExports(manifest), parseGlobals(manifest))). - getOrElse(PublicPackages(List(), List())) - - } catch { - case e : Exception => throw new RuntimeException("Invalid manifest in bundle '%s'".format(jarFile.getPath), e) - } - } - - def bundleSymbolicName(jarFile: File): Option[String] = { - JarFiles.getManifest(jarFile).flatMap(getBundleSymbolicName) - } - - private def parseExportsFromAttribute(manifest : JarManifest, attributeName : String) = { - (for (export <- getMainAttributeValue(manifest, attributeName)) yield - ExportPackageParser.parseAll(export) match { - case noSuccess: ExportPackageParser.NoSuccess => throw new RuntimeException( - "Failed parsing %s: %s".format(attributeName, noSuccess)) - case success => success.get - }). - getOrElse(List()) - } - - private def parseExports = parseExportsFromAttribute(_ : JarManifest, "Export-Package") - - private def parseGlobals(manifest : JarManifest) = { - //TODO: Use separate parser for global packages. - val globals = parseExportsFromAttribute(manifest, "Global-Package") - - if (globals map {_.parameters} exists {!_.isEmpty}) { - throw new RuntimeException("Parameters not valid for Global-Package.") - } - - globals flatMap {_.packageNames} - } - - private def getMainAttributeValue(manifest: JarManifest, name: String): Option[String] = - Option(manifest.getMainAttributes.getValue(name)) - - private def isOsgiManifest = getBundleSymbolicName(_: JarManifest).isDefined - - private def getBundleSymbolicName = getMainAttributeValue(_: JarManifest, "Bundle-SymbolicName") -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/bundle/TransformExportPackages.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/bundle/TransformExportPackages.scala deleted file mode 100644 index f924b12c539..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/bundle/TransformExportPackages.scala +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.bundle - -import java.io.File -import com.yahoo.container.plugin.osgi.ExportPackages.{Export, Parameter} -import com.yahoo.container.plugin.osgi.ExportPackages.Export - -/** - * @author tonytv - */ -object TransformExportPackages extends App { - def replaceVersions(exports: List[Export], newVersion: String): List[Export] = { - mapParameters(exports) { parameters => - parameters map replaceVersion(newVersion) - } - } - - def removeUses(exports: List[Export]): List[Export] = { - mapParameters(exports) { parameters => - parameters filter {_.name != "uses"} - } - } - - def mapParameters(exports: List[Export])(f: List[Parameter] => List[Parameter]): List[Export] = { - exports map { case Export(packageNames: List[String], parameters: List[Parameter]) => - Export(packageNames, f(parameters)) - } - } - - private def replaceVersion(newVersion: String)(parameter: Parameter) = { - parameter match { - case Parameter("version", _) => Parameter("version", newVersion) - case other => other - } - } - - def toExportPackageProperty(exports: List[Export]): String = { - val exportPackages = - exports map { case Export(packageNames: List[String], parameters: List[Parameter]) => - val parameterString = nameEqualsValue(parameters) - (packageNames ++ parameterString) mkString ";" - } - - exportPackages mkString "," - } - - private def nameEqualsValue(parameters: List[Parameter]) = { - parameters map { case Parameter(name, value) => - s"$name=${quote(value)}" - } - } - - def quote(s: String) = '"' + s + '"' -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/Analyze.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/Analyze.scala deleted file mode 100644 index 2f2d034679d..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/Analyze.scala +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import org.objectweb.asm._ -import java.io.{InputStream, File} -import com.yahoo.container.plugin.util.IO.withFileInputStream - -/** - * Main entry point for class analysis - * @author tonytv - */ -object Analyze { - def analyzeClass(classFile : File) : ClassFileMetaData = { - try { - withFileInputStream(classFile) { fileInputStream => - analyzeClass(fileInputStream) - } - } catch { - case e : RuntimeException => throw new RuntimeException("An error occurred when analyzing " + classFile.getPath, e) - } - } - - def analyzeClass(inputStream : InputStream) : ClassFileMetaData = { - val visitor = new AnalyzeClassVisitor() - new ClassReader(inputStream).accept(visitor, ClassReader.SKIP_DEBUG) - visitor.result - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.scala deleted file mode 100644 index 539684f2024..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.scala +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import org.objectweb.asm._ -import com.yahoo.osgi.annotation.{ExportPackage, Version} -import collection.mutable - -/** - * Picks up classes used in class files. - * @author tonytv - */ -private class AnalyzeClassVisitor extends ClassVisitor(Opcodes.ASM6) with AnnotationVisitorTrait with AttributeVisitorTrait { - private var name : String = null - protected val imports : ImportsSet = mutable.Set() - protected var exportPackageAnnotation: Option[ExportPackageAnnotation] = None - - - override def visitAttribute(attribute: Attribute): Unit = super.visitAttribute(attribute) - - override def visitMethod(access: Int, name: String, desc: String, signature: String, - exceptions: Array[String]): MethodVisitor = { - - imports ++= (Type.getReturnType(desc) +: Type.getArgumentTypes(desc)).flatMap(getClassName) - - imports ++= Option(exceptions) getOrElse(Array()) flatMap internalNameToClassName - - AnalyzeSignatureVisitor.analyzeMethod(signature, this) - new AnalyzeMethodVisitor(this) - } - - override def visitField(access: Int, name: String, desc: String, signature: String, value: AnyRef): FieldVisitor = { - imports ++= getClassName(Type.getType(desc)).toList - - AnalyzeSignatureVisitor.analyzeField(signature, this) - new FieldVisitor(Opcodes.ASM6) with SubVisitorTrait with AttributeVisitorTrait with AnnotationVisitorTrait { - val analyzeClassVisitor = AnalyzeClassVisitor.this - - override def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = super.visitAnnotation(desc, visible) - override def visitAttribute(attribute: Attribute): Unit = super.visitAttribute(attribute) - override def visitEnd(): Unit = super.visitEnd() - } - } - - override def visit(version: Int, access: Int, name: String, signature: String, superName: String, interfaces: Array[String]) { - this.name = internalNameToClassName(name).get - - imports ++= (superName +: interfaces) flatMap internalNameToClassName - AnalyzeSignatureVisitor.analyzeClass(signature, this) - } - - override def visitInnerClass(name: String, outerName: String, innerName: String, access: Int) {} - override def visitOuterClass(owner: String, name: String, desc: String) {} - override def visitSource(source: String, debug: String) {} - override def visitEnd() {} - - def addImports(imports: TraversableOnce[String]) { - this.imports ++= imports - } - - override def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = { - if (Type.getType(desc).getClassName == classOf[ExportPackage].getName) { - visitExportPackage() - } else { - super.visitAnnotation(desc, visible) - } - } - - def visitExportPackage(): AnnotationVisitor = { - def defaultVersionValue[T](name: String) = classOf[Version].getMethod(name).getDefaultValue().asInstanceOf[T] - - new AnnotationVisitor(Opcodes.ASM6) { - var major: Int = defaultVersionValue("major") - var minor: Int = defaultVersionValue("minor") - var micro: Int = defaultVersionValue("micro") - var qualifier: String = defaultVersionValue("qualifier") - - override def visit(name: String, value: AnyRef) { - def valueAsInt = value.asInstanceOf[Int] - - name match { - case "major" => major = valueAsInt - case "minor" => minor = valueAsInt - case "micro" => micro = valueAsInt - case "qualifier" => qualifier = value.asInstanceOf[String] - } - } - - override def visitEnd() { - exportPackageAnnotation = Some(ExportPackageAnnotation(major, minor, micro, qualifier)) - } - - override def visitEnum(name: String, desc: String, value: String) {} - override def visitArray(name: String): AnnotationVisitor = this - override def visitAnnotation(name: String, desc: String): AnnotationVisitor = this - } - } - - def result = { - assert(!imports.contains("int")) - new ClassFileMetaData(name, imports.toSet, exportPackageAnnotation) - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.scala deleted file mode 100644 index a8032b6a912..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.scala +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import org.objectweb.asm._ - -/** - * Picks up classes used in method bodies. - * @author tonytv - */ -private class AnalyzeMethodVisitor(val analyzeClassVisitor : AnalyzeClassVisitor) - extends MethodVisitor(Opcodes.ASM6) with AnnotationVisitorTrait with AttributeVisitorTrait with SubVisitorTrait { - - - override def visitParameterAnnotation(parameter: Int, desc: String, visible: Boolean): AnnotationVisitor = super.visitParameterAnnotation(parameter, desc, visible) - override def visitAnnotationDefault(): AnnotationVisitor = super.visitAnnotationDefault() - override def visitAttribute(attribute: Attribute): Unit = super.visitAttribute(attribute) - override def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = super.visitAnnotation(desc, visible) - override def visitEnd(): Unit = super.visitEnd() - - override def visitMultiANewArrayInsn(desc: String, dims: Int) { - imports ++= getClassName(Type.getType(desc)).toList - } - - - override def visitMethodInsn(opcode: Int, owner: String, name: String, desc: String, itf: Boolean) { - imports ++= internalNameToClassName(owner) - imports ++= Type.getArgumentTypes(desc).flatMap(getClassName) - imports ++= getClassName(Type.getReturnType(desc)) - } - - override def visitFieldInsn(opcode: Int, owner: String, name: String, desc: String) { - imports ++= internalNameToClassName(owner) ++ getClassName(Type.getType(desc)).toList - - } - - override def visitTypeInsn(opcode: Int, `type` : String) { - imports ++= internalNameToClassName(`type`) - } - - override def visitTryCatchBlock(start: Label, end: Label, handler: Label, `type` : String) { - if (`type` != null) //null means finally block - imports ++= internalNameToClassName(`type`) - } - - override def visitLocalVariable(name: String, desc: String, signature: String, start: Label, end: Label, index: Int) { - imports += Type.getType(desc).getClassName - } - - override def visitLdcInsn(constant: AnyRef) { - constant match { - case typeConstant: Type => imports ++= getClassName(typeConstant) - case _ => - } - } - - override def visitInvokeDynamicInsn(name: String, desc: String, bootstrapMethod: Handle, bootstrapMethodArgs: AnyRef*) { - bootstrapMethodArgs.foreach { - case typeConstant: Type => - imports ++= getClassName(typeConstant) - case handle: Handle => - imports ++= internalNameToClassName(handle.getOwner) - imports ++= Type.getArgumentTypes(desc).flatMap(getClassName) - case _ : Number => - case _ : String => - case other => throw new AssertionError(s"Unexpected type ${other.getClass} with value '$other'") - } - } - - override def visitMaxs(maxStack: Int, maxLocals: Int) {} - override def visitLineNumber(line: Int, start: Label) {} - //only for debugging - override def visitLookupSwitchInsn(dflt: Label, keys: Array[Int], labels: Array[Label]) {} - - - override def visitTableSwitchInsn(min: Int, max: Int, dflt: Label, labels: Label*): Unit = super.visitTableSwitchInsn(min, max, dflt, labels: _*) - override def visitIincInsn(`var` : Int, increment: Int) {} - override def visitLabel(label: Label) {} - override def visitJumpInsn(opcode: Int, label: Label) {} - override def visitVarInsn(opcode: Int, `var` : Int) {} - override def visitIntInsn(opcode: Int, operand: Int) {} - override def visitInsn(opcode: Int) {} - override def visitFrame(`type` : Int, nLocal: Int, local: Array[AnyRef], nStack: Int, stack: Array[AnyRef]) {} - override def visitCode() {} -} - - - - diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.scala deleted file mode 100644 index 5bb8304cf1e..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.scala +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import org.objectweb.asm.Opcodes -import org.objectweb.asm.signature.{SignatureReader, SignatureVisitor} - - -/** - * @author tonytv - */ - -private class AnalyzeSignatureVisitor(val analyzeClassVisitor: AnalyzeClassVisitor) - extends SignatureVisitor(Opcodes.ASM6) - with SubVisitorTrait { - - - override def visitEnd(): Unit = super.visitEnd() - - override def visitClassType(className: String) { - imports ++= internalNameToClassName(className) - } - - override def visitFormalTypeParameter(name: String) {} - - override def visitClassBound() = this - - override def visitInterfaceBound() = this - - override def visitSuperclass() = this - - override def visitInterface() = this - - override def visitParameterType() = this - - override def visitReturnType() = this - - override def visitExceptionType() = this - - override def visitBaseType(descriptor: Char) {} - - override def visitTypeVariable(name: String) {} - - override def visitArrayType() = this - - override def visitInnerClassType(name: String) {} - - override def visitTypeArgument() {} - - override def visitTypeArgument(wildcard: Char) = this -} - - -object AnalyzeSignatureVisitor { - def analyzeClass(signature: String, analyzeClassVisitor: AnalyzeClassVisitor) { - if (signature != null) { - new SignatureReader(signature).accept(new AnalyzeSignatureVisitor(analyzeClassVisitor)) - } - } - - def analyzeMethod(signature: String, analyzeClassVisitor: AnalyzeClassVisitor) { - analyzeClass(signature, analyzeClassVisitor) - } - - def analyzeField(signature: String, analyzeClassVisitor: AnalyzeClassVisitor) { - if (signature != null) - new SignatureReader(signature).acceptType(new AnalyzeSignatureVisitor(analyzeClassVisitor)) - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnnotationVisitorTrait.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnnotationVisitorTrait.scala deleted file mode 100644 index 0bf6ee4a6b4..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AnnotationVisitorTrait.scala +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import org.objectweb.asm.{Opcodes, AnnotationVisitor, Type} - -/** - * Picks up classes used in annotations. - * @author tonytv - */ -private trait AnnotationVisitorTrait { - protected val imports: ImportsSet - - def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = { - imports ++= getClassName(Type.getType(desc)).toList - - visitAnnotationDefault() - } - - def visitAnnotationDefault(): AnnotationVisitor = - new AnnotationVisitor(Opcodes.ASM6) { - override def visit(name: String, value: AnyRef) {} - - override def visitEnum(name: String, desc: String, value: String) { - imports ++= getClassName(Type.getType(desc)).toList - } - - override def visitArray(name: String): AnnotationVisitor = this - - override def visitAnnotation(name: String, desc: String): AnnotationVisitor = { - imports ++= getClassName(Type.getType(desc)).toList - this - } - - override def visitEnd() {} - } - - def visitParameterAnnotation(parameter: Int, desc: String, visible: Boolean): AnnotationVisitor = - visitAnnotation(desc, visible) -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AttributeVisitorTrait.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AttributeVisitorTrait.scala deleted file mode 100644 index 454bea99f29..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/AttributeVisitorTrait.scala +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import org.objectweb.asm.{Type, Attribute} - -/** - * @author tonytv - */ -private trait AttributeVisitorTrait { - protected val imports: ImportsSet - - def visitAttribute(attribute: Attribute) { - imports ++= getClassName(Type.getObjectType(attribute.`type`)).toList - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.scala deleted file mode 100644 index fb395749895..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.scala +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -/** - * The result of analyzing a .class file. - * @author tonytv - */ -sealed case class ClassFileMetaData(name:String, - referencedClasses : Set[String], - exportPackage : Option[ExportPackageAnnotation]) diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.scala deleted file mode 100644 index f9dc459fcc7..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.scala +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import com.yahoo.container.plugin.util.Strings - -/** - * @author tonytv - */ -case class ExportPackageAnnotation(major: Int, minor: Int, micro: Int, qualifier: String) { - requireNonNegative(major, "major") - requireNonNegative(minor, "minor") - requireNonNegative(micro, "micro") - require(qualifier.matches("""(\p{Alpha}|\p{Digit}|_|-)*"""), - exportPackageError("qualifier must follow the format (alpha|digit|'_'|'-')* but was '%s'.".format(qualifier))) - - - private def requireNonNegative(i: Int, fieldName: String) { - require(i >= 0, exportPackageError("%s must be non-negative but was %d.".format(fieldName, i))) - } - - private def exportPackageError(s: String) = "ExportPackage anntotation: " + s - - def osgiVersion : String = (List(major, minor, micro) ++ Strings.noneIfEmpty(qualifier)).mkString(".") -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/PackageTally.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/PackageTally.scala deleted file mode 100644 index 2d2460cc9fd..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/PackageTally.scala +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import com.yahoo.container.plugin.util.Maps - -/** - * - * @author tonytv - */ -final class PackageTally (private val definedPackagesMap : Map[String, Option[ExportPackageAnnotation]], - referencedPackagesUnfiltered : Set[String]) { - - val referencedPackages = referencedPackagesUnfiltered diff definedPackages - - def definedPackages = definedPackagesMap.keySet - - def exportedPackages = definedPackagesMap collect { case (name, Some(export)) => (name, export) } - - /** - * Represents the classes for two package tallies that are deployed as a single unit. - * - * ExportPackageAnnotations from this has precedence over the other. - */ - def combine(other: PackageTally): PackageTally = { - new PackageTally( - Maps.combine(definedPackagesMap, other.definedPackagesMap)(_ orElse _), - referencedPackages ++ other.referencedPackages) - } -} - - -object PackageTally { - def fromAnalyzedClassFiles(analyzedClassFiles : Seq[ClassFileMetaData]) : PackageTally = { - combine( - for (metaData <- analyzedClassFiles) - yield { - new PackageTally( - Map(Packages.packageName(metaData.name) -> metaData.exportPackage), - metaData.referencedClasses.map(Packages.packageName)) - }) - } - - def combine(packageTallies : Iterable[PackageTally]) : PackageTally = (empty /: packageTallies)(_.combine(_)) - - val empty : PackageTally = new PackageTally(Map(), Set()) -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/Packages.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/Packages.scala deleted file mode 100644 index 517ef2626f9..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/Packages.scala +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -/** - * Utility methods related to packages. - * @author tonytv - */ -object Packages { - case class PackageMetaData(definedPackages: Set[String], referencedExternalPackages: Set[String]) - - def packageName(fullClassName: String) = { - def nullIfNotFound(index : Int) = if (index == -1) 0 else index - - fullClassName.substring(0, nullIfNotFound(fullClassName.lastIndexOf("."))) - } - - - - def analyzePackages(allClasses: Seq[ClassFileMetaData]): PackageMetaData = { - val (definedPackages, referencedClasses) = - (for (classMetaData <- allClasses) - yield (packageName(classMetaData.name), classMetaData.referencedClasses.map(packageName))). - unzip - - PackageMetaData(definedPackages.toSet, referencedClasses.flatten.toSet diff definedPackages.toSet) - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/SubVisitorTrait.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/SubVisitorTrait.scala deleted file mode 100644 index 06ccfa7b88c..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/SubVisitorTrait.scala +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import collection.mutable - -/** - * A visitor that's run for sub construct of a class - * and forwards all its imports the the owning ClassVisitor at the end. - * @author tonytv - */ -private trait SubVisitorTrait { - val analyzeClassVisitor : AnalyzeClassVisitor - - val imports : ImportsSet = mutable.Set() - - def visitEnd() { - analyzeClassVisitor.addImports(imports) - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/package.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/package.scala deleted file mode 100644 index 631884c58e3..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/classanalysis/package.scala +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin - -import org.objectweb.asm.Type -import collection.mutable - -package object classanalysis { - type ImportsSet = mutable.Set[String] - - def internalNameToClassName(internalClassName: String) : Option[String] = { - internalClassName match { - case null => None - case _ => getClassName(Type.getObjectType(internalClassName)) - } - } - - def getClassName(aType: Type): Option[String] = { - import Type._ - - aType.getSort match { - case ARRAY => getClassName(aType.getElementType) - case OBJECT => Some(aType.getClassName) - case _ => None - } - } -} - diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/Artifacts.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/Artifacts.scala deleted file mode 100644 index 8370f8c615e..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/Artifacts.scala +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.mojo - -import org.apache.maven.artifact.Artifact -import org.apache.maven.project.MavenProject - -import scala.collection.JavaConverters._ - -/** - * @author tonytv - */ -object Artifacts { - def getArtifacts(project : MavenProject) = { - type artifactSet = java.util.Set[Artifact] - val artifacts = project.getArtifacts.asInstanceOf[artifactSet].asScala.groupBy(_.getScope) - - def isTypeJar(artifact : Artifact) = artifact.getType == "jar" - def getByScope(scope: String) = - artifacts.getOrElse(scope, Iterable.empty).partition(isTypeJar) - - - val (jarArtifactsToInclude, nonJarArtifactsToInclude) = getByScope(Artifact.SCOPE_COMPILE) - val (jarArtifactsProvided, nonJarArtifactsProvided) = getByScope(Artifact.SCOPE_PROVIDED) - - (jarArtifactsToInclude, jarArtifactsProvided, nonJarArtifactsToInclude ++ nonJarArtifactsProvided) - } - - def getArtifactsToInclude(project: MavenProject) = getArtifacts(project)._1 -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.scala deleted file mode 100644 index 4104e349208..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.scala +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.mojo - -import java.io.File -import java.nio.channels.Channels -import java.util.jar.JarFile -import java.util.zip.ZipEntry - -import com.yahoo.container.plugin.util.{Files, JarFiles} -import org.apache.maven.archiver.{MavenArchiveConfiguration, MavenArchiver} -import org.apache.maven.execution.MavenSession -import org.apache.maven.plugin.AbstractMojo -import org.apache.maven.plugins.annotations.{Mojo, Parameter, ResolutionScope} -import org.apache.maven.project.MavenProject -import org.codehaus.plexus.archiver.jar.JarArchiver - -import scala.collection.JavaConverters._ - -/** - * @author tonytv - */ -@Mojo(name = "assemble-container-plugin", requiresDependencyResolution = ResolutionScope.COMPILE, threadSafe = true) -class AssembleContainerPluginMojo extends AbstractMojo { - object withDependencies - object withoutDependencies - - @Parameter(defaultValue = "${project}") - var project: MavenProject = null - - @Parameter(defaultValue = "${session}", readonly = true, required = true) - var session: MavenSession = null - - @Parameter - var archiveConfiguration: MavenArchiveConfiguration = new MavenArchiveConfiguration - - @Parameter(alias = "UseCommonAssemblyIds", defaultValue = "false") - var useCommonAssemblyIds: Boolean = false - - - def execute() { - val jarSuffixes = - if (useCommonAssemblyIds) Map(withoutDependencies -> ".jar", withDependencies -> "-jar-with-dependencies.jar") - else Map(withoutDependencies -> "-without-dependencies.jar", withDependencies -> "-deploy.jar") - - val jarFiles = jarSuffixes mapValues jarFileInBuildDirectory(build.getFinalName) - - //force recreating the archive - archiveConfiguration.setForced(true) - archiveConfiguration.setManifestFile(new File(new File(project.getBuild.getOutputDirectory), JarFile.MANIFEST_NAME)) - - val jarWithoutDependencies = new JarArchiver() - addClassesDirectory(jarWithoutDependencies) - createArchive(jarFiles(withoutDependencies), jarWithoutDependencies) - project.getArtifact.setFile(jarFiles(withoutDependencies)) - - val jarWithDependencies = new JarArchiver() - addClassesDirectory(jarWithDependencies) - addDependencies(jarWithDependencies) - createArchive(jarFiles(withDependencies), jarWithDependencies) - } - - private def jarFileInBuildDirectory(name: String)(jarSuffix: String) = { - new File(build.getDirectory, name + jarSuffix) - } - - private def addClassesDirectory(jarArchiver: JarArchiver) { - val classesDirectory = new File(build.getOutputDirectory) - if (classesDirectory.isDirectory) { - jarArchiver.addDirectory(classesDirectory) - } - } - - private def createArchive(jarFile: File, jarArchiver: JarArchiver) { - val mavenArchiver = new MavenArchiver - mavenArchiver.setArchiver(jarArchiver) - mavenArchiver.setOutputFile(jarFile) - mavenArchiver.createArchive(session, project, archiveConfiguration) - } - - private def addDependencies(jarArchiver: JarArchiver) { - Artifacts.getArtifactsToInclude(project).foreach { artifact => - if (artifact.getType == "jar") { - jarArchiver.addFile(artifact.getFile, "dependencies/" + artifact.getFile.getName) - copyConfigDefinitions(artifact.getFile, jarArchiver) - } - else - getLog.warn("Unkown artifact type " + artifact.getType) - } - } - - private def copyConfigDefinitions(file: File, jarArchiver: JarArchiver) { - JarFiles.withJarFile(file) { jarFile => - for { - entry <- jarFile.entries().asScala - name = entry.getName - if name.startsWith("configdefinitions/") && name.endsWith(".def") - - } copyConfigDefinition(jarFile, entry, jarArchiver) - } - } - - private def copyConfigDefinition(jarFile: JarFile, entry: ZipEntry, jarArchiver: JarArchiver) { - JarFiles.withInputStream(jarFile, entry) { input => - val defPath = entry.getName.replace("/", File.separator) - val destinationFile = new File(project.getBuild.getOutputDirectory, defPath) - destinationFile.getParentFile.mkdirs() - - Files.withFileOutputStream(destinationFile) { output => - output.getChannel.transferFrom(Channels.newChannel(input), 0, Long.MaxValue) - } - jarArchiver.addFile(destinationFile, entry.getName) - } - } - - private def build = project.getBuild -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.scala deleted file mode 100644 index 2bd146a3920..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.scala +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.mojo - -import java.io.File -import java.nio.file.Paths - -import com.google.common.base.Preconditions -import com.yahoo.container.plugin.bundle.AnalyzeBundle -import com.yahoo.container.plugin.osgi.ProjectBundleClassPaths -import com.yahoo.container.plugin.osgi.ProjectBundleClassPaths.BundleClasspathMapping -import org.apache.maven.artifact.Artifact -import org.apache.maven.plugin.AbstractMojo -import org.apache.maven.plugins.annotations.{Mojo, Parameter, ResolutionScope} -import org.apache.maven.project.MavenProject - -import scala.collection.JavaConverters._ - - - -/** - * Generates mapping from Bundle-SymbolicName to classpath elements, e.g - * myBundle -> List(.m2/repository/com/mylib/Mylib.jar, myBundleProject/target/classes) - * The mapping in stored in a json file. - * @author tonytv - */ -@Mojo(name = "generate-bundle-classpath-mappings", requiresDependencyResolution = ResolutionScope.COMPILE_PLUS_RUNTIME, threadSafe = true) -class GenerateBundleClassPathMappingsMojo extends AbstractMojo { - @Parameter(defaultValue = "${project}") - private var project: MavenProject = null - - //TODO: Combine with com.yahoo.container.plugin.mojo.GenerateOsgiManifestMojo.bundleSymbolicName - @Parameter(alias = "Bundle-SymbolicName", defaultValue = "${project.artifactId}") - private var bundleSymbolicName: String = null - - - /* Sample output -- target/test-classes/bundle-plugin.bundle-classpath-mappings.json - { - "mainBundle": { - "bundleSymbolicName": "bundle-plugin-test", - "classPathElements": [ - "/Users/tonyv/Repos/vespa/bundle-plugin-test/target/classes", - "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/6-SNAPSHOT/jrt-6-SNAPSHOT.jar", - "/Users/tonyv/.m2/repository/com/yahoo/vespa/annotations/6-SNAPSHOT/annotations-6-SNAPSHOT.jar" - ] - }, - "providedDependencies": [ - { - "bundleSymbolicName": "jrt", - "classPathElements": [ - "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/6-SNAPSHOT/jrt-6-SNAPSHOT.jar" - ] - } - ] - } - */ - override def execute(): Unit = { - Preconditions.checkNotNull(bundleSymbolicName) - - val (embeddedArtifacts, providedJarArtifacts, _) = asLists(Artifacts.getArtifacts(project)) - - val embeddedArtifactsFiles = embeddedArtifacts.map(_.getFile) - - val classPathElements = (outputDirectory +: embeddedArtifactsFiles).map(_.getAbsolutePath) - - val classPathMappings = new ProjectBundleClassPaths( - new BundleClasspathMapping(bundleSymbolicName, classPathElements.asJava), - providedJarArtifacts.flatMap(createDependencyClasspathMapping).asJava) - - ProjectBundleClassPaths.save( - testOutputPath.resolve(ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME), - classPathMappings) - } - - private def outputDirectory = new File(project.getBuild.getOutputDirectory) - private def testOutputPath = Paths.get(project.getBuild.getTestOutputDirectory) - - /* TODO: - * 1) add the dependencies of the artifact in the future(i.e. dependencies of dependencies) - * or - * 2) obtain bundles with embedded dependencies from the maven repository, - * and support loading classes from the nested jar files in those bundles. - */ - def createDependencyClasspathMapping(artifact: Artifact): Option[BundleClasspathMapping] = { - for (bundleSymbolicName <- bundleSymbolicNameForArtifact(artifact)) - yield new BundleClasspathMapping(bundleSymbolicName, List(artifact.getFile.getAbsolutePath).asJava) - } - - def bundleSymbolicNameForArtifact(artifact: Artifact): Option[String] = { - if (artifact.getFile.getName.endsWith(".jar")) AnalyzeBundle.bundleSymbolicName(artifact.getFile) - else Some(artifact.getArtifactId) //Not the best heuristic. The other alternatives are parsing the pom file or - //storing information in target/classes when building the provided bundles. - } - - def asLists[A](tuple: (Iterable[A], Iterable[A], Iterable[A])) = - (tuple._1.toList, tuple._2.toList, tuple._3.toList) -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.scala deleted file mode 100644 index 67ce45ed7c6..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.scala +++ /dev/null @@ -1,286 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.mojo - -import java.io.File -import java.util.jar.{Attributes, JarEntry, JarFile} -import java.util.regex.Pattern - -import com.yahoo.container.plugin.bundle.AnalyzeBundle -import com.yahoo.container.plugin.classanalysis.{Analyze, ExportPackageAnnotation, PackageTally} -import com.yahoo.container.plugin.mojo.GenerateOsgiManifestMojo._ -import com.yahoo.container.plugin.osgi.ExportPackages -import com.yahoo.container.plugin.osgi.ExportPackages.Export -import com.yahoo.container.plugin.osgi.ImportPackages.Import -import com.yahoo.container.plugin.osgi.{ExportPackageParser, ExportPackages, ImportPackages} -import com.yahoo.container.plugin.util.Files.allDescendantFiles -import com.yahoo.container.plugin.util.IO.withFileOutputStream -import com.yahoo.container.plugin.util.Iteration.toStream -import com.yahoo.container.plugin.util.JarFiles.{withInputStream, withJarFile} -import com.yahoo.container.plugin.util.Strings -import org.apache.maven.artifact.Artifact -import org.apache.maven.plugin.{AbstractMojo, MojoExecutionException, MojoFailureException} -import org.apache.maven.plugins.annotations.{Mojo, Parameter, ResolutionScope} -import org.apache.maven.project.MavenProject - -import scala.collection.immutable.Map - - -/** - * @author tonytv - */ -@Mojo(name = "generate-osgi-manifest", requiresDependencyResolution = ResolutionScope.TEST, threadSafe = true) -class GenerateOsgiManifestMojo extends AbstractMojo { - - @Parameter(defaultValue = "${project}") - var project: MavenProject = null - - @Parameter - var discApplicationClass: String = null - - @Parameter - var discPreInstallBundle: String = null - - @Parameter(alias = "Bundle-Version", defaultValue = "${project.version}") - var bundleVersion: String = null - - @Parameter(alias = "Bundle-SymbolicName", defaultValue = "${project.artifactId}") - var bundleSymbolicName: String = null - - @Parameter(alias = "Bundle-Activator") - var bundleActivator: String = null - - @Parameter(alias = "X-JDisc-Privileged-Activator") - var jdiscPrivilegedActivator: String = null - - @Parameter(alias = "X-Config-Models") - var configModels: String = null - - @Parameter(alias = "Import-Package") - var importPackage: String = null - - @Parameter(alias = "WebInfUrl") - var webInfUrl: String = null - - @Parameter(alias = "Main-Class") - var mainClass: String = null - - @Parameter(alias = "X-Jersey-Binding") - var jerseyBinding: String = null - - case class PackageInfo(name : String, exportAnnotation : Option[ExportPackageAnnotation]) - - def execute() { - try { - val (jarArtifactsToInclude, jarArtifactsProvided, nonJarArtifacts) = Artifacts.getArtifacts(project) - warnOnUnsupportedArtifacts(nonJarArtifacts) - - val publicPackagesFromProvidedJars = AnalyzeBundle.publicPackagesAggregated(jarArtifactsProvided.map(_.getFile)) - val includedJarPackageTally = definedPackages(jarArtifactsToInclude) - - val projectPackageTally = analyzeProjectClasses() - - val pluginPackageTally = projectPackageTally.combine(includedJarPackageTally) - - warnIfPackagesDefinedOverlapsGlobalPackages(projectPackageTally.definedPackages ++ includedJarPackageTally.definedPackages, - publicPackagesFromProvidedJars.globals) - - if (getLog.isDebugEnabled) { - getLog.debug("Referenced packages = " + pluginPackageTally.referencedPackages) - getLog.debug("Defined packages = " + pluginPackageTally.definedPackages) - getLog.debug("Exported packages of dependencies = " + - publicPackagesFromProvidedJars.exports.map(e => (e.packageNames, e.version getOrElse "")).toSet ) - } - - val calculatedImports = ImportPackages.calculateImports( - pluginPackageTally.referencedPackages, - pluginPackageTally.definedPackages, - ExportPackages.exportsByPackageName(publicPackagesFromProvidedJars.exports)) - - val manualImports = emptyToNone(importPackage) map getManualImports getOrElse Map() - - createManifestFile(new File(project.getBuild.getOutputDirectory), - manifestContent( - project, - jarArtifactsToInclude, - manualImports, - (calculatedImports -- manualImports.keys).values.toSet, - pluginPackageTally)) - - } catch { - case e: MojoFailureException => throw e - case e: MojoExecutionException => throw e - case e: Exception => throw new MojoExecutionException("Failed generating osgi manifest.", e) - } - } - - //TODO: Tell which dependency overlaps - private def warnIfPackagesDefinedOverlapsGlobalPackages(internalPackages: Set[String], globalPackages: List[String]) { - val overlap = internalPackages intersect globalPackages.toSet - if (overlap.nonEmpty) - throw new MojoExecutionException( - "The following packages are both global and included in the bundle:\n%s".format(overlap map (" " + _) mkString ("\n"))) - } - - - def osgiExportPackages(exportedPackages: Map[String, ExportPackageAnnotation]): Iterable[String] = { - for ((name, annotation) <- exportedPackages) - yield name + ";version=" + annotation.osgiVersion - } - - def trimWhitespace(lines: Option[String]): String = { - lines.getOrElse("").split(",").map(_.trim).mkString(",") - } - - def manifestContent(project: MavenProject, jarArtifactsToInclude: Traversable[Artifact], - manualImports: Map[String, Option[String]], imports : Set[Import], - pluginPackageTally : PackageTally) = { - Map[String, String]( - "Created-By" -> "vespa container maven plugin", - "Bundle-ManifestVersion" -> "2", - "Bundle-Name" -> project.getName, - "Bundle-SymbolicName" -> bundleSymbolicName, - "Bundle-Version" -> asBundleVersion(bundleVersion), - "Bundle-Vendor" -> "Yahoo!", - "Bundle-ClassPath" -> bundleClassPath(jarArtifactsToInclude), - "Bundle-Activator" -> bundleActivator, - "X-JDisc-Privileged-Activator" -> jdiscPrivilegedActivator, - "Main-Class" -> mainClass, - "X-JDisc-Application" -> discApplicationClass, - "X-JDisc-Preinstall-Bundle" -> trimWhitespace(Option(discPreInstallBundle)), - "X-Config-Models" -> configModels, - "X-Jersey-Binding" -> jerseyBinding, - "WebInfUrl" -> webInfUrl, - "Import-Package" -> ((manualImports map asOsgiImport) ++ (imports map {_.asOsgiImport})).toList.sorted.mkString(","), - "Export-Package" -> osgiExportPackages(pluginPackageTally.exportedPackages).toList.sorted.mkString(",")) - .filterNot { case (key, value) => value == null || value.isEmpty } - - } - - def asOsgiImport(importSpec: (String, Option[String])) = importSpec match { - case (packageName, Some(version)) => packageName + ";version=" + quote(version) - case (packageName, None) => packageName - } - - def quote(s: String) = '"' + s + '"' - - def createManifestFile(outputDirectory: File, manifestContent: Map[String, String]) { - val manifest = toManifest(manifestContent) - - withFileOutputStream(new File(outputDirectory, JarFile.MANIFEST_NAME)) { - outputStream => - manifest.write(outputStream) - } - } - - def toManifest(manifestContent: Map[String, String]) = { - val manifest = new java.util.jar.Manifest - val mainAttributes = manifest.getMainAttributes - - mainAttributes.put(Attributes.Name.MANIFEST_VERSION, "1.0") - for ((key, value) <- manifestContent) - mainAttributes.putValue(key, value) - - manifest - } - - private def bundleClassPath(artifactsToInclude : Traversable[Artifact]) = - ("." +: artifactsToInclude.map(dependencyPath).toList).mkString(",") - - private def dependencyPath(artifact : Artifact) = - "dependencies/" + artifact.getFile.getName - - private def asBundleVersion(projectVersion: String) = { - require(projectVersion != null, "Missing project version.") - - val parts = projectVersion.split(Pattern.quote("-"), 2) - val numericPart = parts.head.split('.').map(Strings.emptyStringTo("0")).padTo(3, "0").toList - - val majorMinorMicro = numericPart take 3 - majorMinorMicro.mkString(".") - } - - private def warnOnUnsupportedArtifacts(nonJarArtifacts: Traversable[Artifact]) { - val unsupportedArtifacts = nonJarArtifacts.toSet.filter(_.getType != "pom") - - for (artifact <- unsupportedArtifacts) { - getLog.warn(s"Unsupported artifact '${artifact.getId}': Type '${artifact.getType}' is not supported. Please file a feature request.") - } - } - - private def analyzeProjectClasses() : PackageTally = { - val outputDirectory = new File(project.getBuild.getOutputDirectory) - - val analyzedClasses = allDescendantFiles(outputDirectory).filter(file => isClassToAnalyze(file.getName)). - map(Analyze.analyzeClass) - - PackageTally.fromAnalyzedClassFiles(analyzedClasses) - } - - def definedPackages(jarArtifacts: Iterable[Artifact]) : PackageTally = { - PackageTally.combine( - for (jarArtifact <- jarArtifacts) yield { - withJarFile(jarArtifact.getFile) { jarFile => - definedPackages(jarFile) - } - }) - } - - def definedPackages(jarFile: JarFile) = { - val analyzedClasses = - for { - entry <- toStream(jarFile.entries()) - if !entry.isDirectory - if isClassToAnalyze(entry.getName) - metaData = analyzeClass(jarFile, entry) - } yield metaData - - PackageTally.fromAnalyzedClassFiles(analyzedClasses) - } - - def analyzeClass(jarFile : JarFile, entry : JarEntry) = { - try { - withInputStream(jarFile, entry)(Analyze.analyzeClass) - } catch { - case e : Exception => - throw new MojoExecutionException( - "While analyzing the class '%s' in jar file '%s'".format(entry.getName, jarFile.getName), - e) - } - } -} - -object GenerateOsgiManifestMojo { - def getManualImports(importPackage: String): Map[String, Option[String]] = { - try { - (for { - importDirective <- parseImportPackages(importPackage) - packageName <- importDirective.packageNames - } yield packageName -> getVersionThrowOthers(importDirective.parameters)). - toMap - - } catch { - case e: Exception => throw new RuntimeException("Error in Import-Package:" + importPackage, e) - } - } - - def getVersionThrowOthers(parameters: List[ExportPackages.Parameter]): Option[String] = { - parameters match { - case List() => None - case List(ExportPackages.Parameter("version", v)) => Some(v) - case default => throw new RuntimeException("A single, optional version parameter expected, but got " + default) - } - } - - def parseImportPackages(importPackages: String): List[Export] = { - ExportPackageParser.parseAll(importPackages) match { - case ExportPackageParser.NoSuccess(msg, _) => throw new RuntimeException(msg) - case ExportPackageParser.Success(packages, _) => packages - } - } - - def isClassToAnalyze(name: String): Boolean = - name.endsWith(".class") && ! name.endsWith("module-info.class") - - def emptyToNone(str: String) = - Option(str) map {_.trim} filterNot {_.isEmpty} -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/osgi/ExportPackageParser.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/osgi/ExportPackageParser.scala deleted file mode 100644 index 5cd93e84e87..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/osgi/ExportPackageParser.scala +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.osgi - -import scala.util.parsing.combinator.JavaTokenParsers -import ExportPackages.{Parameter, Export} -import com.yahoo.container.plugin.util.Extractors.ListOf -import scala.util.parsing.input.CharSequenceReader -import scala.annotation.tailrec - -/** - * @author tonytv - */ -object ExportPackageParser extends JavaTokenParsers { - val ListOfParameter = new ListOf(classOf[Parameter]) - - - def exportPackage = rep1sep(export, ",") - - //TODO: remove when fix is in current scala library - //Fix for https://github.com/scala/scala-parser-combinators/pull/4 - def stringLiteral_fixed: Parser[String] = ("\""+"""([^"\p{Cntrl}\\]|\\[\\'"bfnrt]|\\u[a-fA-F0-9]{4})*+"""+"\"").r - - @SuppressWarnings(Array("unchecked")) - def export : Parser[Export] = packageName ~ opt(";" ~> (parameters | export)) ^^ { - case (packageName : String) ~ optional => { - optional match { - case None => Export(List(packageName.asInstanceOf[String]), List()) - case Some(e: Export) => e.copy(packageNames = packageName +: e.packageNames) - case Some(ListOfParameter(parameters)) => Export(List(packageName), parameters) - } - } - } - - def parameters = rep1sep(parameter, ";") - - def parameter = (directive | attribute) ^^ { - case k ~ v => Parameter(k.toString, v.toString) - } - - def directive = (extended_ <~ ":=") ~ argument - def attribute = (extended_ <~ "=") ~ argument - - def packageName = rep1sep(ident_, ".") ^^ { - x => x.mkString(".") - } - - def extended = rep1("""\p{Alnum}""".r | "_" | "-" | ".") ^^ { - _.mkString - } - - def argument = (extended_ | stringLiteral_ | failure("argument expected")) ^^ { - val quote = '"'.toString - _.toString.stripPrefix(quote).stripSuffix(quote) - } - - def parseAll(in: CharSequence): ParseResult[List[Export]] = { - try { - parseAll(exportPackage, in) - } catch { - case e: StackOverflowError => - throw new RuntimeException("Failed parsing Export-Package: '''\n" + in + "\n'''", e) - } - } - - //*** For debugging StackOverflow error **/ - def ident_ = printStackOverflow(ident)("ident") - def stringLiteral_ = printStackOverflow(stringLiteral_fixed)("stringLiteral_fixed") - def extended_ = printStackOverflow(extended)("extended") - - def printStackOverflow[T](p: => Parser[T])(name: String): Parser[T] = Parser{ in => - try { - p(in) - } catch { - case e: StackOverflowError => - val input = in match { - case reader: CharSequenceReader => readerToString(reader) - case other => other.toString - } - println(s"***StackOverflow for $name with input '''$input'''") - throw e - } - } - - @tailrec - def readerToString(reader: CharSequenceReader, current: String = ""): String = { - if (reader.atEnd) current - else readerToString(reader.rest, current + reader.first) - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/osgi/ExportPackages.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/osgi/ExportPackages.scala deleted file mode 100644 index 4a973c0b9b1..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/osgi/ExportPackages.scala +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.osgi - -/** - * @author tonytv - */ -object ExportPackages { - - case class Export(packageNames: List[String], parameters: List[Parameter]) { - def version: Option[String] = { - (for ( - param <- parameters if param.name == "version" - ) yield param.value). - headOption - } - } - - case class Parameter(name: String, value: String) - - def exportsByPackageName(exports: Seq[Export]): Map[String, Export] = { - (for { - export <- exports.reverse //ensure that earlier exports of a package overrides later exports. - packageName <- export.packageNames - } yield packageName -> export). - toMap - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/osgi/ImportPackages.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/osgi/ImportPackages.scala deleted file mode 100644 index b6b47b954c0..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/osgi/ImportPackages.scala +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.osgi - -import ExportPackages.Export -import util.control.Exception - -/** - * @author tonytv - */ -object ImportPackages { - case class Import(packageName : String, version : Option[String]) { - val majorMinorMicroVersion = Exception.handling(classOf[NumberFormatException]). - by( e => throw new IllegalArgumentException( - "Invalid version number '%s' for package '%s'.".format(version.get, packageName), e)) { - - version map { _.split('.') take 3 map {_.toInt} } - } - - def majorVersion = majorMinorMicroVersion map { _.head } - - // TODO: Detecting guava packages should be based on Bundle-SymbolicName, not package name. - def importVersionRange = { - def upperLimit = - if (isGuavaPackage) InfiniteVersion // guava increases major version for each release - else majorVersion.get + 1 - - version map (v => "[%s,%s)".format(majorMinorMicroVersion.get.mkString("."), upperLimit)) - } - - def isGuavaPackage = packageName.equals(GuavaBasePackage) || packageName.startsWith(GuavaBasePackage + ".") - - def asOsgiImport = packageName + (importVersionRange map {";version=\"" + _ + '"'} getOrElse("")) - } - - - val GuavaBasePackage = "com.google.common" - val InfiniteVersion = 99999 - - def calculateImports(referencedPackages : Set[String], - implementedPackages : Set[String], - exportedPackages : Map[String, Export]) : Map[String, Import] = { - (for { - undefinedPackage <- referencedPackages diff implementedPackages - export <- exportedPackages.get(undefinedPackage) - } yield undefinedPackage -> Import(undefinedPackage, version(export)))( - collection.breakOut) - } - - def version(export: Export): Option[String] = - export.parameters.find(_.name == "version").map(_.value) -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Extractors.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Extractors.scala deleted file mode 100644 index 83f23f905fb..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Extractors.scala +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.util - -/** -* @author tonytv -*/ -object Extractors { - class ListOf[C](val c : Class[C]) { - def unapply[X](xs : X) : Option[List[C]] = { - xs match { - case x :: xr if c.isInstance(x) => unapply(xr) map ( c.cast(x) :: _) - case Nil => Some(Nil) - case _ => None - } - } - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Files.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Files.scala deleted file mode 100644 index d32e57784da..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Files.scala +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.util - -import java.io.{FileOutputStream, File} -import com.yahoo.container.plugin.util.IO._ - -/** - * @author tonytv - */ -object Files { - def allDescendantFiles(file: File): Stream[File] = { - if (file.isFile) - Stream(file) - else if (file.isDirectory) - file.listFiles().toStream.map(allDescendantFiles).flatten - else - Stream.empty - } - - def withFileOutputStream[T](file: File)(f: FileOutputStream => T): T = { - using(new FileOutputStream(file), readOnly = false)(f) - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/IO.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/IO.scala deleted file mode 100644 index 90f3eb3e7fd..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/IO.scala +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.util - -import java.io.{Closeable, FileOutputStream, OutputStream, FileInputStream, File} -import util.control.Exception -import scala.Either - -/** Utility methods relating to IO - * @author tonytv - */ -object IO { - def withFileInputStream[T](file : File)(f : FileInputStream => T) = { - using(new FileInputStream(file), readOnly = true)(f) - } - - /** - * Creates a new file and all it's parent directories, - * and provides a file output stream to the file. - * - * Exceptions from closing have priority over exceptions from f. - */ - def withFileOutputStream[T](file: File)(f: OutputStream => T) { - makeDirectoriesRecursive(file.getParentFile) - using(new FileOutputStream(file), readOnly = false )(f) - } - - def makeDirectoriesRecursive(file: File) { - if (!file.mkdirs() && !file.isDirectory) { - throw new RuntimeException("Could not create directory " + file.getPath) - } - } - - def using[RESOURCE <: Closeable, T](resource : RESOURCE, readOnly : Boolean)(f : RESOURCE => T) : T = { - def catchPromiscuously = Exception.catchingPromiscuously(classOf[Throwable]) - - val resultOrException = catchPromiscuously either f(resource) - val closeException = Exception.allCatch either resource.close() - - prioritizeFirstException( - resultOrException, - if (readOnly) Right(()) else closeException) fold (throw _, identity) - } - - private def prioritizeFirstException[T](first: Either[Throwable, T], second: Either[Throwable, Unit]) = - first fold ( Left(_), value => second fold ( Left(_), _ => Right(value) ) ) -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Iteration.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Iteration.scala deleted file mode 100644 index fa50f5cdd17..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Iteration.scala +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.util - - /** - * @author tonytv - */ -object Iteration { - def toStream[T](enumeration: java.util.Enumeration[T]): Stream[T] = { - if (enumeration.hasMoreElements) - Stream.cons(enumeration.nextElement(), toStream(enumeration)) - else - Stream.Empty - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/JarFiles.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/JarFiles.scala deleted file mode 100644 index d7578f2b338..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/JarFiles.scala +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.util - -import java.util.jar.JarFile -import java.util.zip.{ZipFile, ZipEntry} -import IO.using -import java.io.{Closeable, InputStream, File} - -/** - * @author tonytv - */ -object JarFiles { - def withJarFile[T](file : File)(f : JarFile => T ) : T = - using(new JarFile(file) with Closeable, readOnly = true)(f) - - def withInputStream[T](zipFile: ZipFile, zipEntry: ZipEntry)(f: InputStream => T): T = - using(zipFile.getInputStream(zipEntry), readOnly = true)(f) - - def getManifest(jarFile : File) : Option[java.util.jar.Manifest] = { - withJarFile(jarFile) { jar => - Option(jar.getManifest) - } - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Maps.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Maps.scala deleted file mode 100644 index cafc8431b0e..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Maps.scala +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.util - -import collection.mutable.MultiMap - -/** - * @author tonytv - */ -object Maps { - def combine[K, V](map1 : Map[K, V], map2 : Map[K, V])(f : (V, V) => V) : Map[K, V] = { - def logicError : V = throw new RuntimeException("Logic error.") - def combineValues(key : K) = key -> f(map1.getOrElse(key, logicError), map2.getOrElse(key, logicError)) - - val keysInBoth = map1.keySet intersect map2.keySet - def notInBoth = !keysInBoth.contains(_ : K) - - map1.filterKeys(notInBoth) ++ map2.filterKeys(notInBoth) ++ keysInBoth.map(combineValues) - } -} diff --git a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Strings.scala b/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Strings.scala deleted file mode 100644 index 5f854fda944..00000000000 --- a/bundle-plugin/src/main/scala/com/yahoo/container/plugin/util/Strings.scala +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.util - - /** - * @author tonytv - */ -object Strings { - def emptyStringTo(replacement: String)(s: String) = { - if (s.isEmpty) replacement - else s - } - - def noneIfEmpty(s: String) = Option(s).filterNot(_.isEmpty) -} diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.java new file mode 100644 index 00000000000..8cccf0598ab --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.java @@ -0,0 +1,85 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.bundle; + +import com.yahoo.container.plugin.bundle.AnalyzeBundle.PublicPackages; +import com.yahoo.container.plugin.osgi.ExportPackages; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static com.yahoo.container.plugin.classanalysis.TestUtilities.throwableMessage; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.Assert.assertThat; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class AnalyzeBundleTest { + private final List<ExportPackages.Export> exports; + private final Map<String, ExportPackages.Export> exportsByPackageName; + + File jarDir = new File("src/test/resources/jar"); + + public AnalyzeBundleTest() { + File notOsgi = new File(jarDir, "notAOsgiBundle.jar"); + File simple = new File(jarDir, "simple1.jar"); + PublicPackages pp = AnalyzeBundle.publicPackagesAggregated(Arrays.asList(notOsgi, simple)); + this.exports = pp.exports; + this.exportsByPackageName = ExportPackages.exportsByPackageName(exports); + } + + private File jarFile(String name) { + return new File(jarDir, name); + } + + @Test + public void require_that_non_osgi_bundles_are_ignored() { + assertThat(exportsByPackageName.keySet(), not(hasItem("com.yahoo.sample.exported.package.ignored"))); + } + + @Test + public void require_that_exports_are_retrieved_from_manifest_in_jars() { + assertThat(exportsByPackageName.keySet().size(), is(1)); + assertThat(exportsByPackageName.keySet(), hasItem("com.yahoo.sample.exported.package")); + } + + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void require_that_invalid_exports_throws_exception() { + exception.expect(Exception.class); + + exception.expectMessage(containsString("Invalid manifest in bundle")); + exception.expectMessage(matchesPattern("Invalid manifest in bundle '.*errorExport.jar'")); + exception.expectCause(throwableMessage(startsWith("Failed parsing Export-Package"))); + + AnalyzeBundle.publicPackages(jarFile("errorExport.jar")); + } + + private TypeSafeMatcher<String> matchesPattern(String pattern) { + return new TypeSafeMatcher<String>() { + @Override + protected boolean matchesSafely(String s) { + return s.matches(pattern); + } + + @Override + public void describeTo(Description description) { + description.appendText("expects String that matches the pattern ").appendValue(pattern); + } + }; + } +} diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java new file mode 100644 index 00000000000..aba6e8f14e8 --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java @@ -0,0 +1,167 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import com.yahoo.container.plugin.classanalysis.sampleclasses.Base; +import com.yahoo.container.plugin.classanalysis.sampleclasses.ClassAnnotation; +import com.yahoo.container.plugin.classanalysis.sampleclasses.Derived; +import com.yahoo.container.plugin.classanalysis.sampleclasses.DummyAnnotation; +import com.yahoo.container.plugin.classanalysis.sampleclasses.Fields; +import com.yahoo.container.plugin.classanalysis.sampleclasses.Interface1; +import com.yahoo.container.plugin.classanalysis.sampleclasses.Interface2; +import com.yahoo.container.plugin.classanalysis.sampleclasses.MethodAnnotation; +import com.yahoo.container.plugin.classanalysis.sampleclasses.MethodInvocation; +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import javax.security.auth.login.LoginException; +import java.awt.Image; +import java.awt.image.ImagingOpException; +import java.awt.image.Kernel; +import java.util.Optional; + +import static com.yahoo.container.plugin.classanalysis.TestUtilities.analyzeClass; +import static com.yahoo.container.plugin.classanalysis.TestUtilities.classFile; +import static com.yahoo.container.plugin.classanalysis.TestUtilities.name; +import static com.yahoo.container.plugin.classanalysis.TestUtilities.throwableMessage; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertThat; + +/** + * Tests that analysis of class files works. + * + * @author Tony Vaagenes + * @author ollivir + */ +public class AnalyzeClassTest { + @Test + public void require_that_full_class_name_is_returned() { + assertThat(analyzeClass(Base.class).getName(), is(name(Base.class))); + } + + @Test + public void require_that_base_class_is_included() { + assertThat(analyzeClass(Derived.class).getReferencedClasses(), hasItem(name(Base.class))); + } + + @Test + public void require_that_implemented_interfaces_are_included() { + assertThat(analyzeClass(Base.class).getReferencedClasses(), + allOf(hasItem(name(Interface1.class)), hasItem(name(Interface2.class)))); + } + + @Test + public void require_that_interface_can_be_analyzed() { + ClassFileMetaData classMetaData = analyzeClass(Interface1.class); + + assertThat(classMetaData.getName(), is(name(Interface1.class))); + assertThat(classMetaData.getReferencedClasses(), hasItem(name(Interface2.class))); + } + + @Test + public void require_that_return_type_is_included() { + assertThat(analyzeClass(Interface1.class).getReferencedClasses(), hasItem(name(Image.class))); + } + + @Test + public void require_that_parameters_are_included() { + assertThat(analyzeClass(Interface1.class).getReferencedClasses(), hasItem(name(Kernel.class))); + } + + @Test + public void require_that_exceptions_are_included() { + assertThat(analyzeClass(Interface1.class).getReferencedClasses(), hasItem(name(ImagingOpException.class))); + } + + @Test + public void require_that_basic_types_ignored() { + assertThat(analyzeClass(Interface1.class).getReferencedClasses(), not(anyOf(hasItem("int"), hasItem("float")))); + } + + @Test + public void require_that_arrays_of_basic_types_ignored() { + assertThat(analyzeClass(Interface1.class).getReferencedClasses(), not(anyOf(hasItem("int[]"), hasItem("int[][]")))); + } + + @Test + public void require_that_instance_field_types_are_included() { + assertThat(analyzeClass(Fields.class).getReferencedClasses(), hasItem(name(String.class))); + } + + @Test + public void require_that_static_field_types_are_included() { + assertThat(analyzeClass(Fields.class).getReferencedClasses(), hasItem(name(java.util.List.class))); + } + + @Test + public void require_that_field_annotation_is_included() { + assertThat(analyzeClass(Fields.class).getReferencedClasses(), hasItem(name(DummyAnnotation.class))); + } + + @Test + public void require_that_class_annotation_is_included() { + assertThat(analyzeClass(ClassAnnotation.class).getReferencedClasses(), hasItem(name(DummyAnnotation.class))); + } + + @Test + public void require_that_method_annotation_is_included() { + assertThat(analyzeClass(MethodAnnotation.class).getReferencedClasses(), hasItem(name(DummyAnnotation.class))); + } + + @Test + public void require_that_export_package_annotations_are_ignored() { + assertThat(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")) + .getReferencedClasses(), not(anyOf(hasItem(name(ExportPackage.class)), hasItem(name(Version.class))))); + } + + @Test + public void require_that_export_annotations_are_processed() { + assertThat( + Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")).getExportPackage(), + is(Optional.of(new ExportPackageAnnotation(3, 1, 4, "TEST_QUALIFIER-2")))); + } + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void require_that_export_annotations_are_validated() { + expectedException.expect(RuntimeException.class); + expectedException.expectMessage(containsString("invalid/package-info")); + expectedException.expectCause(throwableMessage(containsString("qualifier must follow the format"))); + expectedException.expectCause(throwableMessage(containsString("'EXAMPLE INVALID QUALIFIER'"))); + + Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.invalid.package-info")); + } + + @Test + public void require_that_catch_clauses_are_included() { + assertThat(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.CatchException")) + .getReferencedClasses(), hasItem(name(LoginException.class))); + } + + @Test + public void require_that_class_references_are_included() { + assertThat(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.ClassReference")) + .getReferencedClasses(), hasItem(name(Interface1.class))); + } + + @Test + public void require_that_return_type_of_method_invocations_are_included() { + assertThat(analyzeClass(MethodInvocation.class).getReferencedClasses(), hasItem(name(Image.class))); + } + + @Test + public void require_that_attributes_are_included() { + //Uses com/coremedia/iso/Utf8.class from com.googlecode.mp4parser:isoparser:1.0-RC-1 + assertThat(Analyze.analyzeClass(classFile("class/Utf8")).getReferencedClasses(), + hasItem("org.aspectj.weaver.MethodDeclarationLineNumber")); + } +} diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java new file mode 100644 index 00000000000..d722f1eba2d --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java @@ -0,0 +1,73 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import com.yahoo.container.plugin.classanalysis.sampleclasses.Base; +import com.yahoo.container.plugin.classanalysis.sampleclasses.ClassWithMethod; +import com.yahoo.container.plugin.classanalysis.sampleclasses.Derived; +import com.yahoo.container.plugin.classanalysis.sampleclasses.Dummy; +import com.yahoo.container.plugin.classanalysis.sampleclasses.Fields; +import com.yahoo.container.plugin.classanalysis.sampleclasses.Interface1; +import com.yahoo.container.plugin.classanalysis.sampleclasses.Methods; +import org.junit.Test; + +import java.io.PrintStream; + +import static com.yahoo.container.plugin.classanalysis.TestUtilities.analyzeClass; +import static com.yahoo.container.plugin.classanalysis.TestUtilities.name; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertThat; + +/** + * Tests that classes used in method bodies are included in the imports list. + * + * @author tonytv + */ +public class AnalyzeMethodBodyTest { + @Test + public void require_that_class_of_locals_are_included() { + assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Base.class))); + } + + @Test + public void require_that_class_of_locals_in_static_method_are_included() { + assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Derived.class))); + } + + @Test + public void require_that_field_references_are_included() { + assertThat(analyzeClass(Methods.class).getReferencedClasses(), + allOf(hasItem(name(java.util.List.class)), hasItem(name(Fields.class)))); + } + + @Test + public void require_that_class_owning_field_is_included() { + assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(System.class))); + } + + @Test + public void require_that_class_containing_method_is_included() { + assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(PrintStream.class))); + } + + @Test + public void require_that_element_of_new_multidimensional_array_is_included() { + assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Interface1.class))); + } + + @Test + public void require_that_basic_arrays_are_not_included() { + assertThat(analyzeClass(Methods.class).getReferencedClasses(), not(hasItem("int[]"))); + } + + @Test + public void require_that_container_generic_parameters_are_included() { + assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Dummy.class))); + } + + @Test + public void require_that_class_owning_method_handler_is_included() { + assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(ClassWithMethod.class))); + } +} diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java new file mode 100644 index 00000000000..6ff6bafd816 --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java @@ -0,0 +1,40 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis; + +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; + +import java.io.File; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class TestUtilities { + public static ClassFileMetaData analyzeClass(Class<?> clazz) { + return Analyze.analyzeClass(classFile(name(clazz))); + } + + public static File classFile(String className) { + return new File("target/test-classes/" + className.replace('.', '/') + ".class"); + } + + public static String name(Class<?> clazz) { + return clazz.getName(); + } + + public static TypeSafeMatcher<Throwable> throwableMessage(final Matcher<String> matcher) { + return new TypeSafeMatcher<Throwable>() { + @Override + protected boolean matchesSafely(Throwable throwable) { + return matcher.matches(throwable.getMessage()); + } + + @Override + public void describeTo(Description description) { + description.appendText("expects Throwable and a message matching ").appendDescriptionOf(matcher); + } + }; + } +} diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/ClassReference.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/ClassReference.java index b4a4f90398f..f650f0f0c7f 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/ClassReference.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/ClassReference.java @@ -6,6 +6,7 @@ package com.yahoo.container.plugin.classanalysis.sampleclasses; */ public class ClassReference { void classReference() { + @SuppressWarnings("unused") Class<?> clazz = Interface1.class; } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/MethodInvocation.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/MethodInvocation.java index 992799e3032..a576c764eec 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/MethodInvocation.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/MethodInvocation.java @@ -7,6 +7,7 @@ package com.yahoo.container.plugin.classanalysis.sampleclasses; public class MethodInvocation { void invokeMethod() { Interface1 interface1 = null; + @SuppressWarnings({ "unused", "null" }) Object o = interface1.methodSignatureTest(null, null); } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java index 568a5070961..abe325768fc 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java @@ -8,6 +8,7 @@ import java.util.Map; * Input for class analysis tests. * @author tonytv */ +@SuppressWarnings("unused") public class Methods { public void method1() { Base b = new Base(); diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ExportPackageParserTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ExportPackageParserTest.java new file mode 100644 index 00000000000..d869b8ec4d9 --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ExportPackageParserTest.java @@ -0,0 +1,295 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.osgi; + +import com.yahoo.container.plugin.osgi.ExportPackages.Export; +import com.yahoo.container.plugin.osgi.ExportPackages.Parameter; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Test; + +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; +import static org.junit.Assert.assertThat; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class ExportPackageParserTest { + private final Parameter versionParameter = new Parameter("version", "1.2.3.sample"); + + @Test + public void require_that_package_is_parsed_correctly() { + List<Export> exports = ExportPackageParser.parseExports("sample.exported.package"); + + assertThat(exports.size(), is(1)); + assertThat(exports.get(0).getParameters(), empty()); + assertThat(exports.get(0).getPackageNames(), contains("sample.exported.package")); + } + + @Test + public void require_that_version_is_parsed_correctly() { + List<Export> exports = ExportPackageParser.parseExports("com.yahoo.sample.exported.package;version=\"1.2.3.sample\""); + + assertThat(exports.size(), is(1)); + Export export = exports.get(0); + assertThat(export.getPackageNames(), contains("com.yahoo.sample.exported.package")); + assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + } + + @Test + public void require_that_multiple_packages_with_same_parameters_is_parsed_correctly() { + List<Export> exports = ExportPackageParser.parseExports("exported.package1;exported.package2;version=\"1.2.3.sample\""); + + assertThat(exports.size(), is(1)); + Export export = exports.get(0); + assertThat(export.getPackageNames(), contains("exported.package1", "exported.package2")); + assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + } + + @Test + public void require_that_spaces_between_separators_are_allowed() { + List<Export> exports = ExportPackageParser.parseExports("exported.package1 , exported.package2 ; version = \"1.2.3.sample\" "); + + assertThat(exports.size(), is(2)); + Export export = exports.get(0); + assertThat(export.getPackageNames(), contains("exported.package1")); + export = exports.get(1); + assertThat(export.getPackageNames(), contains("exported.package2")); + assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + } + + @SuppressWarnings("unchecked") + @Test + public void require_that_multiple_parameters_for_a_package_is_parsed_correctly() { + List<Export> exports = ExportPackageParser.parseExports("exported.package;version=\"1.2.3.sample\";param2=true"); + + assertThat(exports.size(), is(1)); + Export export = exports.get(0); + assertThat(export.getParameters(), contains(parameterMatching(versionParameter), parameterMatching("param2", "true"))); + } + + @Test + public void require_that_multiple_exports_are_parsed_correctly() { + List<Export> exports = ExportPackageParser.parseExports("exported.package1,exported.package2"); + assertThat(exports.size(), is(2)); + Export export = exports.get(0); + assertThat(export.getPackageNames(), contains("exported.package1")); + assertThat(export.getParameters(), empty()); + export = exports.get(1); + assertThat(export.getPackageNames(), contains("exported.package2")); + assertThat(export.getParameters(), empty()); + + exports = ExportPackageParser.parseExports("exported.package1;version=\"1.2.3.sample\",exported.package2"); + assertThat(exports.size(), is(2)); + export = exports.get(0); + assertThat(export.getPackageNames(), contains("exported.package1")); + assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + export = exports.get(1); + assertThat(export.getPackageNames(), contains("exported.package2")); + assertThat(export.getParameters(), empty()); + + exports = ExportPackageParser.parseExports("exported.package1,exported.package2;version=\"1.2.3.sample\""); + assertThat(exports.size(), is(2)); + export = exports.get(0); + assertThat(export.getPackageNames(), contains("exported.package1")); + assertThat(export.getParameters(), empty()); + export = exports.get(1); + assertThat(export.getPackageNames(), contains("exported.package2")); + assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + } + + @Test + public void require_that_long_string_literals_do_not_cause_stack_overflow_error() { + //From jersey-server-1.13.jar + String exportHeader = "com.sun.jersey.server.impl.wadl;uses:=\"com.sun.jersey.api.model,com.sun.resea" + + "rch.ws.wadl,com.sun.jersey.api.wadl.config,com.sun.jersey.server.wadl,com.sun." + + "jersey.api.core,javax.xml.bind,javax.ws.rs.core,com.sun.jersey.server.impl.uri" + + ",com.sun.jersey.core.spi.factory,com.sun.jersey.server.impl.model.method,com.s" + + "un.jersey.api.uri,com.sun.jersey.core.header,com.sun.jersey.spi.dispatch,javax" + + ".ws.rs,com.sun.jersey.spi.resource\";version=\"1.13.0\",com.sun.jersey.server." + + "impl.model.parameter.multivalued;uses:=\"com.sun.jersey.spi,javax.ws.rs.core,c" + + "om.sun.jersey.api.container,com.sun.jersey.impl,javax.xml.parsers,org.xml.sax," + + "javax.xml.transform,javax.xml.bind.annotation,javax.xml.transform.sax,com.sun." + + "jersey.spi.inject,javax.xml.bind,javax.ws.rs.ext,com.sun.jersey.api.model,com." + + "sun.jersey.core.reflection,javax.ws.rs,com.sun.jersey.core.spi.component,com.s" + + "un.jersey.core.header\";version=\"1.13.0\",com.sun.jersey.server.impl.model.pa" + + "rameter;uses:=\"com.sun.jersey.api.model,com.sun.jersey.core.spi.component,com" + + ".sun.jersey.server.impl.model.parameter.multivalued,com.sun.jersey.spi.inject," + + "com.sun.jersey.api,com.sun.jersey.api.core,com.sun.jersey.server.impl.inject,j" + + "avax.ws.rs.core,javax.ws.rs,com.sun.jersey.core.header,com.sun.jersey.api.repr" + + "esentation\";version=\"1.13.0\",com.sun.jersey.server.impl.application;uses:=" + + "\"com.sun.jersey.core.spi.component,com.sun.jersey.api.core,com.sun.jersey.spi" + + ",com.sun.jersey.spi.inject,javax.ws.rs.core,com.sun.jersey.api.container,javax" + + ".ws.rs.ext,com.sun.jersey.spi.container,com.sun.jersey.core.reflection,com.sun" + + ".jersey.api.model,com.sun.jersey.impl,com.sun.jersey.spi.dispatch,com.sun.jers" + + "ey.server.impl.model,com.sun.jersey.server.impl.wadl,com.sun.jersey.server.imp" + + "l.uri,com.sun.jersey.core.spi.factory,com.sun.jersey.api.uri,com.sun.jersey.se" + + "rver.impl.uri.rules,com.sun.jersey.spi.uri.rules,com.sun.jersey.server.spi.com" + + "ponent,com.sun.jersey.core.util,com.sun.jersey.core.header,com.sun.jersey.core" + + ".spi.component.ioc,javax.ws.rs,com.sun.jersey.server.impl,com.sun.jersey.serve" + + "r.wadl,com.sun.jersey.server.impl.inject,com.sun.jersey.server.impl.component," + + "com.sun.jersey.spi.monitoring,com.sun.jersey.server.impl.monitoring,com.sun.je" + + "rsey.api.container.filter,com.sun.jersey.server.impl.model.parameter.multivalu" + + "ed,com.sun.jersey.server.impl.model.parameter,com.sun.jersey.server.impl.templ" + + "ate,com.sun.jersey.spi.template,com.sun.jersey.server.impl.resource,com.sun.je" + + "rsey.server.impl.modelapi.annotation,com.sun.jersey.server.impl.container.filt" + + "er,com.sun.jersey.server.impl.modelapi.validation,com.sun.jersey.api,com.sun.j" + + "ersey.spi.service\";version=\"1.13.0\",com.sun.jersey.server.impl.component;us" + + "es:=\"com.sun.jersey.api.model,com.sun.jersey.core.spi.component,com.sun.jerse" + + "y.server.spi.component,com.sun.jersey.api.core,com.sun.jersey.core.spi.compone" + + "nt.ioc,com.sun.jersey.server.impl.inject,com.sun.jersey.server.impl.resource,c" + + "om.sun.jersey.api.container,com.sun.jersey.core.reflection,com.sun.jersey.spi." + + "inject\";version=\"1.13.0\",com.sun.jersey.server.impl.provider;uses:=\"com.su" + + "n.jersey.core.spi.factory,com.sun.jersey.api.container,com.sun.jersey.api.core" + + ",javax.ws.rs.core\";version=\"1.13.0\",com.sun.jersey.server.impl.template;use" + + "s:=\"com.sun.jersey.core.spi.component,com.sun.jersey.api.view,com.sun.jersey." + + "spi.template,javax.ws.rs.core,com.sun.jersey.core.header,com.sun.jersey.server" + + ".impl.model.method,com.sun.jersey.spi.dispatch,com.sun.jersey.api.uri,javax.ws" + + ".rs,com.sun.jersey.spi.inject,javax.ws.rs.ext,com.sun.jersey.server.impl.uri.r" + + "ules,com.sun.jersey.server.probes,com.sun.jersey.core.reflection,com.sun.jerse" + + "y.spi.uri.rules,com.sun.jersey.spi.container,com.sun.jersey.api.core\";version" + + "=\"1.13.0\",com.sun.jersey.server.osgi;uses:=\"com.sun.jersey.server.impl.prov" + + "ider,org.osgi.framework,javax.ws.rs.ext\";version=\"1.13.0\",com.sun.jersey.se" + + "rver.wadl.generators.resourcedoc.model;uses:=\"javax.xml.bind.annotation,javax" + + ".xml.namespace\";version=\"1.13.0\",com.sun.jersey.server.impl.resource;uses:=" + + "\"com.sun.jersey.api.model,com.sun.jersey.core.spi.component,com.sun.jersey.se" + + "rver.spi.component,com.sun.jersey.api.core,com.sun.jersey.api.container,javax." + + "ws.rs,com.sun.jersey.server.impl.inject,com.sun.jersey.core.spi.component.ioc," + + "javax.ws.rs.core\";version=\"1.13.0\",com.sun.jersey.server.impl.monitoring;us" + + "es:=\"com.sun.jersey.spi.monitoring,com.sun.jersey.spi.service,com.sun.jersey." + + "api.model,com.sun.jersey.spi.container,javax.ws.rs.ext,com.sun.jersey.core.spi" + + ".component\";version=\"1.13.0\",com.sun.jersey.server.impl.modelapi.annotation" + + ";uses:=\"com.sun.jersey.api.model,javax.ws.rs.core,javax.ws.rs,com.sun.jersey." + + "core.reflection,com.sun.jersey.core.header,com.sun.jersey.impl\";version=\"1.1" + + "3.0\",com.sun.jersey.server.impl.container;uses:=\"com.sun.jersey.server.impl." + + "application,com.sun.jersey.spi.container,com.sun.jersey.api.container\";versio" + + "n=\"1.13.0\",com.sun.jersey.server.wadl;uses:=\"javax.ws.rs.core,com.sun.resea" + + "rch.ws.wadl,javax.xml.namespace,com.sun.jersey.api.model,javax.xml.bind,javax." + + "ws.rs,com.sun.jersey.server.wadl.generators,com.sun.jersey.server.impl.modelap" + + "i.annotation,com.sun.jersey.server.impl\";version=\"1.13.0\",com.sun.jersey.se" + + "rver.impl.model.method.dispatch;uses:=\"com.sun.jersey.api.model,com.sun.jerse" + + "y.api.core,com.sun.jersey.spi.container,com.sun.jersey.server.impl.inject,com." + + "sun.jersey.api,javax.ws.rs.core,com.sun.jersey.core.spi.factory,com.sun.jersey" + + ".spi.inject,com.sun.jersey.spi.dispatch,com.sun.jersey.core.spi.component,java" + + "x.ws.rs,com.sun.jersey.server.impl.model.parameter.multivalued,com.sun.jersey." + + "api.representation,com.sun.jersey.api.container\";version=\"1.13.0\",com.sun.j" + + "ersey.server.impl;uses:=\"javax.naming,com.sun.jersey.api.core,com.sun.jersey." + + "core.header,javax.ws.rs.core,com.sun.jersey.server.impl.model,com.sun.jersey.s" + + "pi.container\";version=\"1.13.0\",com.sun.jersey.server.wadl.generators.resour" + + "cedoc;uses:=\"com.sun.jersey.api.model,com.sun.jersey.server.wadl.generators.r" + + "esourcedoc.model,com.sun.jersey.server.wadl.generators.resourcedoc.xhtml,com.s" + + "un.research.ws.wadl,javax.xml.namespace,com.sun.jersey.server.wadl,javax.xml.b" + + "ind,javax.ws.rs.core\";version=\"1.13.0\",com.sun.jersey.server.impl.container" + + ".httpserver;uses:=\"com.sun.net.httpserver,com.sun.jersey.spi.container,javax." + + "ws.rs.core,com.sun.jersey.core.header,com.sun.jersey.api.container,com.sun.jer" + + "sey.core.util,com.sun.jersey.api.core\";version=\"1.13.0\",com.sun.jersey.serv" + + "er.impl.container.filter;uses:=\"com.sun.jersey.api.model,com.sun.jersey.spi.c" + + "ontainer,com.sun.jersey.core.spi.component,com.sun.jersey.api.core,javax.ws.rs" + + ",com.sun.jersey.server.impl.uri,javax.ws.rs.core\";version=\"1.13.0\",com.sun." + + "jersey.server.wadl.generators.resourcedoc.xhtml;uses:=\"javax.xml.bind,javax.x" + + "ml.namespace,javax.xml.bind.annotation\";version=\"1.13.0\",com.sun.jersey.ser" + + "ver.impl.uri.rules;uses:=\"com.sun.jersey.spi.uri.rules,com.sun.jersey.api.uri" + + ",com.sun.jersey.api.core,com.sun.jersey.server.impl.model.method,com.sun.jerse" + + "y.spi.dispatch,com.sun.jersey.core.header,javax.ws.rs.core,com.sun.jersey.api." + + "model,com.sun.jersey.server.probes,com.sun.jersey.core.reflection,com.sun.jers" + + "ey.server.impl.template,com.sun.jersey.spi.monitoring,com.sun.jersey.api,com.s" + + "un.jersey.spi.container,com.sun.jersey.server.impl.uri,javax.ws.rs,com.sun.jer" + + "sey.api.container,com.sun.jersey.server.impl.inject,com.sun.jersey.spi.inject," + + "com.sun.jersey.server.impl.uri.rules.automata\";version=\"1.13.0\",com.sun.jer" + + "sey.server.spi.component;uses:=\"com.sun.jersey.spi.inject,com.sun.jersey.api." + + "model,com.sun.jersey.core.spi.component,com.sun.jersey.api.core,com.sun.jersey" + + ".server.impl.inject,com.sun.jersey.api.container,com.sun.jersey.core.spi.compo" + + "nent.ioc\";version=\"1.13.0\",com.sun.jersey.server.probes;version=\"1.13.0\"," + + "com.sun.jersey.server.wadl.generators;uses:=\"com.sun.research.ws.wadl,javax.x" + + "ml.bind.annotation,com.sun.jersey.api.model,com.sun.jersey.server.wadl,javax.x" + + "ml.bind,javax.ws.rs.core,com.sun.jersey.api,javax.xml.namespace,javax.xml.tran" + + "sform,javax.xml.transform.stream\";version=\"1.13.0\",com.sun.jersey.server.im" + + "pl.modelapi.validation;uses:=\"com.sun.jersey.api.model,javax.ws.rs,com.sun.je" + + "rsey.impl,com.sun.jersey.api.core,com.sun.jersey.core.reflection,javax.ws.rs.c" + + "ore\";version=\"1.13.0\",com.sun.jersey.server.impl.model.method;uses:=\"com.s" + + "un.jersey.api.container,com.sun.jersey.spi.dispatch,com.sun.jersey.api.uri,com" + + ".sun.jersey.api.model,com.sun.jersey.server.impl.container.filter,com.sun.jers" + + "ey.impl,com.sun.jersey.spi.container,com.sun.jersey.spi.inject,com.sun.jersey." + + "api.core,javax.ws.rs.core,com.sun.jersey.core.header\";version=\"1.13.0\",com." + + "sun.jersey.server.impl.model;uses:=\"javax.ws.rs,com.sun.jersey.impl,com.sun.j" + + "ersey.api.container,com.sun.jersey.core.header,com.sun.jersey.core.header.read" + + "er,com.sun.jersey.api.core,javax.ws.rs.core,com.sun.jersey.server.impl.model.m" + + "ethod,com.sun.jersey.server.impl.container.filter,com.sun.jersey.api.model,com" + + ".sun.jersey.server.impl.wadl,com.sun.jersey.spi.monitoring,com.sun.jersey.serv" + + "er.impl.uri,com.sun.jersey.spi.container,com.sun.jersey.server.impl.inject,com" + + ".sun.jersey.spi.inject,com.sun.jersey.api.uri,com.sun.jersey.core.spi.componen" + + "t,com.sun.jersey.server.impl.uri.rules,com.sun.jersey.server.impl.template,com" + + ".sun.jersey.api.view,com.sun.jersey.spi.uri.rules\";version=\"1.13.0\",com.sun" + + ".jersey.server.impl.uri.rules.automata;uses:=\"com.sun.jersey.server.impl.uri," + + "com.sun.jersey.spi.uri.rules,com.sun.jersey.server.impl.uri.rules,com.sun.jers" + + "ey.api.uri\";version=\"1.13.0\",com.sun.jersey.server.impl.uri;uses:=\"com.sun" + + ".jersey.api.uri,javax.ws.rs.core\";version=\"1.13.0\",com.sun.jersey.server.im" + + "pl.inject;uses:=\"com.sun.jersey.api.core,com.sun.jersey.spi.inject,javax.ws.r" + + "s,com.sun.jersey.api.container,com.sun.jersey.api.model,com.sun.jersey.core.sp" + + "i.component,com.sun.jersey.core.spi.factory\";version=\"1.13.0\",com.sun.jerse" + + "y.spi.scanning;uses:=\"org.objectweb.asm,com.sun.jersey.core.reflection,com.su" + + "n.jersey.core.spi.scanning,javax.ws.rs,javax.ws.rs.ext\";version=\"1.13.0\",co" + + "m.sun.jersey.spi.resource;uses:=\"com.sun.jersey.server.impl.resource,com.sun." + + "jersey.server.spi.component\";version=\"1.13.0\",com.sun.jersey.spi.template;u" + + "ses:=\"com.sun.jersey.api.view,javax.ws.rs.core,com.sun.jersey.api.container\"" + + ";version=\"1.13.0\",com.sun.jersey.spi.dispatch;uses:=\"com.sun.jersey.api.cor" + + "e\";version=\"1.13.0\",com.sun.jersey.spi.uri.rules;uses:=\"com.sun.jersey.api" + + ".core,com.sun.jersey.api.model,com.sun.jersey.spi.container,com.sun.jersey.api" + + ".uri\";version=\"1.13.0\",com.sun.jersey.spi.container;uses:=\"javax.ws.rs,com" + + ".sun.jersey.api.representation,com.sun.jersey.core.header,com.sun.jersey.spi,j" + + "avax.ws.rs.core,com.sun.jersey.api.container,com.sun.jersey.api.core,com.sun.j" + + "ersey.core.util,com.sun.jersey.core.header.reader,com.sun.jersey.server.impl,c" + + "om.sun.jersey.core.reflection,javax.ws.rs.ext,com.sun.jersey.server.impl.model" + + ",com.sun.jersey.api,com.sun.jersey.api.uri,com.sun.jersey.core.spi.factory,com" + + ".sun.jersey.spi.monitoring,com.sun.jersey.api.model,com.sun.jersey.core.spi.co" + + "mponent,com.sun.jersey.server.impl.application,com.sun.jersey.impl,com.sun.jer" + + "sey.spi.inject,com.sun.jersey.spi.dispatch,com.sun.jersey.server.impl.inject,c" + + "om.sun.jersey.core.spi.component.ioc,com.sun.jersey.spi.service\";version=\"1." + + "13.0\",com.sun.jersey.spi.monitoring;uses:=\"com.sun.jersey.api.model,com.sun." + + "jersey.spi.container,javax.ws.rs.ext\";version=\"1.13.0\",com.sun.jersey.api;u" + + "ses:=\"javax.ws.rs,javax.ws.rs.core,com.sun.jersey.core.header,com.sun.jersey." + + "core.spi.factory\";version=\"1.13.0\",com.sun.jersey.api.core;uses:=\"javax.ws" + + ".rs.core,com.sun.jersey.core.spi.scanning,com.sun.jersey.api.model,com.sun.jer" + + "sey.api.uri,javax.ws.rs,com.sun.jersey.core.header,com.sun.jersey.api.represen" + + "tation,com.sun.jersey.core.util,javax.ws.rs.ext,com.sun.jersey.api.container,c" + + "om.sun.jersey.spi.scanning,com.sun.jersey.spi.container,com.sun.jersey.server." + + "impl.application\";version=\"1.13.0\",com.sun.jersey.api.wadl.config;uses:=\"c" + + "om.sun.jersey.server.wadl,com.sun.jersey.api.core,com.sun.jersey.core.reflecti" + + "on,com.sun.jersey.server.wadl.generators\";version=\"1.13.0\",com.sun.jersey.a" + + "pi.model;uses:=\"javax.ws.rs.core,com.sun.jersey.spi.container\";version=\"1.1" + + "3.0\",com.sun.jersey.api.view;version=\"1.13.0\",com.sun.jersey.api.container." + + "filter;uses:=\"javax.ws.rs,com.sun.jersey.spi.container,javax.ws.rs.core,com.s" + + "un.jersey.api.container,com.sun.jersey.api.core,com.sun.jersey.core.util,com.s" + + "un.jersey.core.header,com.sun.jersey.api.representation,com.sun.jersey.api.mod" + + "el,javax.annotation.security\";version=\"1.13.0\",com.sun.jersey.api.container" + + ";uses:=\"com.sun.jersey.api.core,com.sun.jersey.spi.container,com.sun.jersey.s" + + "pi.service,com.sun.jersey.core.spi.component.ioc\";version=\"1.13.0\",com.sun." + + "jersey.api.container.httpserver;uses:=\"com.sun.net.httpserver,com.sun.jersey." + + "api.core,com.sun.jersey.api.container,com.sun.jersey.core.spi.component.ioc\";" + + "version=\"1.13.0\",com.sun.research.ws.wadl;uses:=\"javax.xml.bind.annotation," + + "javax.xml.bind.annotation.adapters,javax.xml.namespace\";version=\"1.13.0\""; + ExportPackageParser.parseExports(exportHeader); + } + + private static TypeSafeMatcher<Parameter> parameterMatching(final String name, final String value) { + return new TypeSafeMatcher<Parameter>() { + @Override + protected boolean matchesSafely(Parameter parameter) { + return parameter.getName().equals(name) && parameter.getValue().equals(value); + } + + @Override + public void describeTo(Description description) { + description.appendText("Parameter with name ").appendValue(name).appendText(" with value ").appendValue(value); + } + }; + } + + private static TypeSafeMatcher<Parameter> parameterMatching(final Parameter param) { + return parameterMatching(param.getName(), param.getValue()); + } +} diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java new file mode 100644 index 00000000000..23960323a31 --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java @@ -0,0 +1,130 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.osgi; + +import com.yahoo.container.plugin.osgi.ExportPackages.Export; +import com.yahoo.container.plugin.osgi.ExportPackages.Parameter; +import com.yahoo.container.plugin.osgi.ImportPackages.Import; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static java.util.Collections.singletonList; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class ImportPackageTest { + private static final String PACKAGE_NAME = "com.yahoo.exported"; + private Set<String> referencedPackages = Collections.singleton(PACKAGE_NAME); + private Map<String, Export> exports = exportByPackageName(new Export(singletonList(PACKAGE_NAME), Collections.emptyList())); + private Map<String, Export> exportsWithVersion = exportByPackageName( + new Export(singletonList(PACKAGE_NAME), singletonList(new Parameter("version", "1.3")))); + + private static Map<String, Export> exportByPackageName(Export export) { + return ExportPackages.exportsByPackageName(singletonList(export)); + } + + @Test + public void require_that_non_implemented_import_with_matching_export_is_included() { + Set<Import> imports = calculateImports(referencedPackages, emptySet(), exports); + assertThat(imports, contains(importMatching(PACKAGE_NAME, ""))); + } + + @Test + public void require_that_non_implemented_import_without_matching_export_is_excluded() { + Set<Import> imports = calculateImports(referencedPackages, emptySet(), emptyMap()); + assertThat(imports, empty()); + } + + @Test + public void require_that_implemented_import_with_matching_export_is_excluded() { + Set<Import> imports = calculateImports(referencedPackages, referencedPackages, exports); + + assertThat(imports, empty()); + } + + @Test + public void require_that_version_is_included() { + Set<Import> imports = calculateImports(referencedPackages, emptySet(), exportsWithVersion); + + assertThat(imports, contains(importMatching(PACKAGE_NAME, "1.3"))); + } + + @Test + public void require_that_all_versions_up_to_the_next_major_version_is_in_range() { + assertThat(new Import("foo", Optional.of("1.2")).importVersionRange().get(), is("[1.2,2)")); + } + + // TODO: Detecting guava packages should be based on bundle-symbolicName, not package name. + @Test + public void require_that_for_guava_all_future_major_versions_are_in_range() { + Optional<String> rangeWithInfiniteUpperLimit = Optional.of("[18.1," + ImportPackages.INFINITE_VERSION + ")"); + assertThat(new Import("com.google.common", Optional.of("18.1")).importVersionRange(), is(rangeWithInfiniteUpperLimit)); + assertThat(new Import("com.google.common.foo", Optional.of("18.1")).importVersionRange(), is(rangeWithInfiniteUpperLimit)); + assertThat(new Import("com.google.commonality", Optional.of("18.1")).importVersionRange(), is(Optional.of("[18.1,19)"))); + } + + @Test + public void require_that_none_version_gives_non_version_range() { + assertThat(new Import("foo", Optional.empty()).importVersionRange(), is(Optional.empty())); + } + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void require_that_exception_is_thrown_when_major_component_is_non_numeric() { + expectedException.expect(IllegalArgumentException.class); + new Import("foo", Optional.of("1notValid.2")); + } + + @Test + public void require_that_osgi_import_supports_missing_version() { + assertThat(new Import("com.yahoo.exported", Optional.empty()).asOsgiImport(), is("com.yahoo.exported")); + } + + @Test + public void require_that_osgi_import_version_range_includes_all_versions_from_the_current_up_to_the_next_major_version() { + assertThat(new Import("com.yahoo.exported", Optional.of("1.2")).asOsgiImport(), is("com.yahoo.exported;version=\"[1.2,2)\"")); + } + + @Test + public void require_that_osgi_import_version_range_ignores_qualifier() { + assertThat(new Import("com.yahoo.exported", Optional.of("1.2.3.qualifier")).asOsgiImport(), + is("com.yahoo.exported;version=\"[1.2.3,2)\"")); + } + + private static Set<Import> calculateImports(Set<String> referencedPackages, Set<String> implementedPackages, + Map<String, Export> exportedPackages) { + return new HashSet<>(ImportPackages.calculateImports(referencedPackages, implementedPackages, exportedPackages).values()); + } + + private static TypeSafeMatcher<Import> importMatching(String packageName, String version) { + return new TypeSafeMatcher<Import>() { + @Override + protected boolean matchesSafely(Import anImport) { + return anImport.packageName().equals(packageName) && anImport.version().equals(version); + } + + @Override + public void describeTo(Description description) { + description.appendText("an Import of package ").appendValue(packageName).appendText(" with version ").appendValue(version); + } + }; + } +} diff --git a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.scala b/bundle-plugin/src/test/scala/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.scala deleted file mode 100644 index 3ef1b1fea87..00000000000 --- a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.scala +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.bundle - -import org.scalatest.junit.{AssertionsForJUnit, JUnitSuite} -import org.junit.Test -import com.yahoo.container.plugin.bundle.AnalyzeBundle.PublicPackages -import com.yahoo.container.plugin.osgi.ExportPackages -import java.io.File - -import org.scalatest.Matchers - -/** - * @author tonytv - */ -class AnalyzeBundleTest extends JUnitSuite with AssertionsForJUnit with Matchers { - val jarDir = new File("src/test/resources/jar") - - val PublicPackages(exports, globals) = AnalyzeBundle.publicPackagesAggregated( - List("notAOsgiBundle.jar", "simple1.jar").map(jarFile)) - - val exportsByPackageName = ExportPackages.exportsByPackageName(exports) - - def jarFile(name : String) : File = new File(jarDir, name) - - @Test - def require_that_non_osgi_bundles_are_ignored() { - exportsByPackageName should not contain key("com.yahoo.sample.exported.package.ignored") - } - - @Test - def require_that_exports_are_retrieved_from_manifest_in_jars() { - exportsByPackageName.keySet should be (Set("com.yahoo.sample.exported.package")) - } - - @Test - def require_that_invalid_exports_throws_exception() { - val exception = intercept[Exception](AnalyzeBundle.publicPackages(jarFile("errorExport.jar"))) - exception.getMessage should startWith regex ("Invalid manifest in bundle '.*errorExport.jar'") - exception.getCause.getMessage should startWith ("Failed parsing Export-Package") - } -} diff --git a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.scala b/bundle-plugin/src/test/scala/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.scala deleted file mode 100644 index e2d0e3c6f10..00000000000 --- a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.scala +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import org.junit.Test -import org.scalatest.junit.{AssertionsForJUnit, JUnitSuite} -import java.awt.Image -import java.awt.image.{ImagingOpException, Kernel} - -import sampleclasses._ -import TestUtilities._ -import com.yahoo.osgi.annotation.{ExportPackage, Version} -import javax.security.auth.login.LoginException - -import org.scalatest.Matchers - -/** - * Tests that analysis of class files works. - * @author tonytv - */ -class AnalyzeClassTest extends JUnitSuite with AssertionsForJUnit with Matchers { - @Test def require_that_full_class_name_is_returned() { - analyzeClass[Base].name should be(name[Base]) - } - - @Test def require_that_base_class_is_included() { - analyzeClass[Derived].referencedClasses should contain (name[Base]) - } - - @Test def require_that_implemented_interfaces_are_included() { - analyzeClass[Base].referencedClasses should (contain (name[Interface1]) and contain (name[Interface2])) - } - - @Test def require_that_interface_can_be_analyzed() { - val classMetaData = analyzeClass[Interface1] - - classMetaData.name should be(name[Interface1]) - classMetaData.referencedClasses should contain(name[Interface2]) - } - - @Test def require_that_return_type_is_included() { - analyzeClass[Interface1].referencedClasses should contain (name[Image]) - } - - @Test def require_that_parameters_are_included() { - analyzeClass[Interface1].referencedClasses should contain (name[Kernel]) - } - - @Test def require_that_exceptions_are_included() { - analyzeClass[Interface1].referencedClasses should contain (name[ImagingOpException]) - } - - @Test def require_that_basic_types_ignored() { - analyzeClass[Interface1].referencedClasses should not (contain ("int") or contain ("float")) - } - - @Test def require_that_arrays_of_basic_types_ignored() { - analyzeClass[Interface1].referencedClasses should not (contain ("int[]") or contain ("int[][]")) - } - - @Test def require_that_instance_field_types_are_included() { - analyzeClass[Fields].referencedClasses should contain (name[String]) - } - - @Test def require_that_static_field_types_are_included() { - analyzeClass[Fields].referencedClasses should contain (name[java.util.List[_]]) - } - - @Test def require_that_field_annotation_is_included() { - analyzeClass[Fields].referencedClasses should contain (name[DummyAnnotation]) - } - - @Test def require_that_class_annotation_is_included() { - analyzeClass[ClassAnnotation].referencedClasses should contain(name[DummyAnnotation]) - } - - @Test def require_that_method_annotation_is_included() { - analyzeClass[MethodAnnotation].referencedClasses should contain(name[DummyAnnotation]) - } - - @Test def require_that_export_package_annotations_are_ignored() { - Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")). - referencedClasses should not (contain (name[ExportPackage]) or contain (name[Version])) - } - - @Test def require_that_export_annotations_are_processed() { - Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")). - exportPackage should be (Some(ExportPackageAnnotation(3, 1, 4, "TEST_QUALIFIER-2"))) - } - - @Test def require_that_export_annotations_are_validated() { - val exception = intercept[RuntimeException] { - Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.invalid.package-info")) - } - - exception.getMessage should include ("invalid/package-info") - exception.getCause.getMessage should include ("qualifier must follow the format") - exception.getCause.getMessage should include ("'EXAMPLE INVALID QUALIFIER'") - } - - @Test def require_that_catch_clauses_are_included() { - Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.CatchException")). - referencedClasses should contain(name[LoginException]) - } - - @Test def require_that_class_references_are_included() { - Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.ClassReference")). - referencedClasses should contain(name[Interface1]) - } - - @Test def require_that_return_type_of_method_invocations_are_included() { - analyzeClass[MethodInvocation].referencedClasses should contain(name[Image]) - } - - @Test def require_that_attributes_are_included() { - //Uses com/coremedia/iso/Utf8.class from com.googlecode.mp4parser:isoparser:1.0-RC-1 - Analyze.analyzeClass(classFile("class/Utf8")).referencedClasses should contain("org.aspectj.weaver.MethodDeclarationLineNumber") - } -} diff --git a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.scala b/bundle-plugin/src/test/scala/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.scala deleted file mode 100644 index 43f52884f39..00000000000 --- a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.scala +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import org.scalatest.junit.{AssertionsForJUnit, JUnitSuite} -import sampleclasses._ -import TestUtilities._ -import org.junit.Test -import java.io.PrintStream - -import org.scalatest.Matchers - -/** - * Tests that classes used in method bodies are included in the imports list. - * @author tonytv - */ -class AnalyzeMethodBodyTest extends JUnitSuite with AssertionsForJUnit with Matchers { - @Test def require_that_class_of_locals_are_included() { - analyzeClass[Methods].referencedClasses should contain(name[Base]) - } - - @Test def require_that_class_of_locals_in_static_method_are_included() { - analyzeClass[Methods].referencedClasses should contain(name[Derived]) - } - - @Test def require_that_field_references_are_included() { - analyzeClass[Methods].referencedClasses should (contain (name[java.util.List[_]]) and contain (name[Fields])) - } - - @Test def require_that_class_owning_field_is_included() { - analyzeClass[Methods].referencedClasses should contain (name[System]) - } - - @Test def require_that_class_containing_method_is_included() { - analyzeClass[Methods].referencedClasses should contain (name[PrintStream]) - } - - @Test def require_that_element_of_new_multidimensional_array_is_included() { - analyzeClass[Methods].referencedClasses should contain (name[Interface1]) - } - - @Test def require_that_basic_arrays_are_not_included() { - analyzeClass[Methods].referencedClasses should not (contain("int[]")) - } - - @Test def require_that_container_generic_parameters_are_included() { - analyzeClass[Methods].referencedClasses should contain(name[Dummy]) - } - - @Test def require_that_class_owning_method_handler_is_included() { - analyzeClass[Methods].referencedClasses should contain(name[ClassWithMethod]) - } -} diff --git a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/classanalysis/TestUtilities.scala b/bundle-plugin/src/test/scala/com/yahoo/container/plugin/classanalysis/TestUtilities.scala deleted file mode 100644 index fbc3554e9e9..00000000000 --- a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/classanalysis/TestUtilities.scala +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.classanalysis - -import scala.reflect.ClassTag -import java.io.File - -/** - * @author tonytv - */ -object TestUtilities { - def analyzeClass[T](implicit m: ClassTag[T]) = - Analyze.analyzeClass(classFile(name(m))) - - def classFile(className : String) = - new File("target/test-classes/" + className.replace('.', '/') + ".class") - - def name[T](implicit m: ClassTag[T]) = m.runtimeClass.getName -} diff --git a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/osgi/ExportPackageParserTest.scala b/bundle-plugin/src/test/scala/com/yahoo/container/plugin/osgi/ExportPackageParserTest.scala deleted file mode 100644 index 5ddc7fef16d..00000000000 --- a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/osgi/ExportPackageParserTest.scala +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.osgi - -import org.scalatest.junit.{AssertionsForJUnit, JUnitSuite} -import org.junit.Test -import ExportPackages.{Export, Parameter} -import org.scalatest.Matchers - -/** - * @author tonytv - */ -class ExportPackageParserTest extends JUnitSuite with AssertionsForJUnit with Matchers { - val versionParameter = Parameter("version", "1.2.3.sample") - - @Test - def require_that_package_is_parsed_correctly() { - ExportPackageParser.parseAll("sample.exported.package").get should - be (List(Export(List("sample.exported.package"), List()))) - } - - @Test - def require_that_version_is_parsed_correctly() { - ExportPackageParser.parseAll("com.yahoo.sample.exported.package;version=\"1.2.3.sample\"").get should - be (List(Export(List("com.yahoo.sample.exported.package"), List(versionParameter)))) - } - - @Test - def require_that_multiple_packages_with_same_parameters_is_parsed_correctly() { - ExportPackageParser.parseAll("exported.package1;exported.package2;version=\"1.2.3.sample\"").get should - be (List(Export(List("exported.package1", "exported.package2"), List(versionParameter)))) - } - - @Test - def require_that_multiple_parameters_for_a_package_is_parsed_correctly() { - ExportPackageParser.parseAll("exported.package;version=\"1.2.3.sample\";param2=true").get.head.parameters should - be (List(versionParameter, Parameter("param2", "true"))) - } - - @Test - def require_that_multiple_exports_are_parsed_correctly() { - ExportPackageParser.parseAll("exported.package1,exported.package2").get should - be(List( - Export(List("exported.package1"), List()), - Export(List("exported.package2"), List()))) - - ExportPackageParser.parseAll("exported.package1;version=\"1.2.3.sample\",exported.package2").get should - be (List( - Export(List("exported.package1"), List(versionParameter)), - Export(List("exported.package2"), List()))) - - ExportPackageParser.parseAll("exported.package1,exported.package2;version=\"1.2.3.sample\"").get should - be(List( - Export(List("exported.package1"), List()), - Export(List("exported.package2"), List(versionParameter)))) - } - - @Test - def require_that_long_string_literals_do_not_cause_stack_overflow_error() { - //From jersey-server-1.13.jar - val export = """com.sun.jersey.server.impl.wadl;uses:="com.sun.jersey.api.model,com.sun.research.ws.wadl,com.sun.jersey.api.wadl.config,com.sun.jersey.server.wadl,com.sun.jersey.api.core,javax.xml.bind,javax.ws.rs.core,com.sun.jersey.server.impl.uri,com.sun.jersey.core.spi.factory,com.sun.jersey.server.impl.model.method,com.sun.jersey.api.uri,com.sun.jersey.core.header,com.sun.jersey.spi.dispatch,javax.ws.rs,com.sun.jersey.spi.resource";version="1.13.0",com.sun.jersey.server.impl.model.parameter.multivalued;uses:="com.sun.jersey.spi,javax.ws.rs.core,com.sun.jersey.api.container,com.sun.jersey.impl,javax.xml.parsers,org.xml.sax,javax.xml.transform,javax.xml.bind.annotation,javax.xml.transform.sax,com.sun.jersey.spi.inject,javax.xml.bind,javax.ws.rs.ext,com.sun.jersey.api.model,com.sun.jersey.core.reflection,javax.ws.rs,com.sun.jersey.core.spi.component,com.sun.jersey.core.header";version="1.13.0",com.sun.jersey.server.impl.model.parameter;uses:="com.sun.jersey.api.model,com.sun.jersey.core.spi.component,com.sun.jersey.server.impl.model.parameter.multivalued,com.sun.jersey.spi.inject,com.sun.jersey.api,com.sun.jersey.api.core,com.sun.jersey.server.impl.inject,javax.ws.rs.core,javax.ws.rs,com.sun.jersey.core.header,com.sun.jersey.api.representation";version="1.13.0",com.sun.jersey.server.impl.application;uses:="com.sun.jersey.core.spi.component,com.sun.jersey.api.core,com.sun.jersey.spi,com.sun.jersey.spi.inject,javax.ws.rs.core,com.sun.jersey.api.container,javax.ws.rs.ext,com.sun.jersey.spi.container,com.sun.jersey.core.reflection,com.sun.jersey.api.model,com.sun.jersey.impl,com.sun.jersey.spi.dispatch,com.sun.jersey.server.impl.model,com.sun.jersey.server.impl.wadl,com.sun.jersey.server.impl.uri,com.sun.jersey.core.spi.factory,com.sun.jersey.api.uri,com.sun.jersey.server.impl.uri.rules,com.sun.jersey.spi.uri.rules,com.sun.jersey.server.spi.component,com.sun.jersey.core.util,com.sun.jersey.core.header,com.sun.jersey.core.spi.component.ioc,javax.ws.rs,com.sun.jersey.server.impl,com.sun.jersey.server.wadl,com.sun.jersey.server.impl.inject,com.sun.jersey.server.impl.component,com.sun.jersey.spi.monitoring,com.sun.jersey.server.impl.monitoring,com.sun.jersey.api.container.filter,com.sun.jersey.server.impl.model.parameter.multivalued,com.sun.jersey.server.impl.model.parameter,com.sun.jersey.server.impl.template,com.sun.jersey.spi.template,com.sun.jersey.server.impl.resource,com.sun.jersey.server.impl.modelapi.annotation,com.sun.jersey.server.impl.container.filter,com.sun.jersey.server.impl.modelapi.validation,com.sun.jersey.api,com.sun.jersey.spi.service";version="1.13.0",com.sun.jersey.server.impl.component;uses:="com.sun.jersey.api.model,com.sun.jersey.core.spi.component,com.sun.jersey.server.spi.component,com.sun.jersey.api.core,com.sun.jersey.core.spi.component.ioc,com.sun.jersey.server.impl.inject,com.sun.jersey.server.impl.resource,com.sun.jersey.api.container,com.sun.jersey.core.reflection,com.sun.jersey.spi.inject";version="1.13.0",com.sun.jersey.server.impl.provider;uses:="com.sun.jersey.core.spi.factory,com.sun.jersey.api.container,com.sun.jersey.api.core,javax.ws.rs.core";version="1.13.0",com.sun.jersey.server.impl.template;uses:="com.sun.jersey.core.spi.component,com.sun.jersey.api.view,com.sun.jersey.spi.template,javax.ws.rs.core,com.sun.jersey.core.header,com.sun.jersey.server.impl.model.method,com.sun.jersey.spi.dispatch,com.sun.jersey.api.uri,javax.ws.rs,com.sun.jersey.spi.inject,javax.ws.rs.ext,com.sun.jersey.server.impl.uri.rules,com.sun.jersey.server.probes,com.sun.jersey.core.reflection,com.sun.jersey.spi.uri.rules,com.sun.jersey.spi.container,com.sun.jersey.api.core";version="1.13.0",com.sun.jersey.server.osgi;uses:="com.sun.jersey.server.impl.provider,org.osgi.framework,javax.ws.rs.ext";version="1.13.0",com.sun.jersey.server.wadl.generators.resourcedoc.model;uses:="javax.xml.bind.annotation,javax.xml.namespace";version="1.13.0",com.sun.jersey.server.impl.resource;uses:="com.sun.jersey.api.model,com.sun.jersey.core.spi.component,com.sun.jersey.server.spi.component,com.sun.jersey.api.core,com.sun.jersey.api.container,javax.ws.rs,com.sun.jersey.server.impl.inject,com.sun.jersey.core.spi.component.ioc,javax.ws.rs.core";version="1.13.0",com.sun.jersey.server.impl.monitoring;uses:="com.sun.jersey.spi.monitoring,com.sun.jersey.spi.service,com.sun.jersey.api.model,com.sun.jersey.spi.container,javax.ws.rs.ext,com.sun.jersey.core.spi.component";version="1.13.0",com.sun.jersey.server.impl.modelapi.annotation;uses:="com.sun.jersey.api.model,javax.ws.rs.core,javax.ws.rs,com.sun.jersey.core.reflection,com.sun.jersey.core.header,com.sun.jersey.impl";version="1.13.0",com.sun.jersey.server.impl.container;uses:="com.sun.jersey.server.impl.application,com.sun.jersey.spi.container,com.sun.jersey.api.container";version="1.13.0",com.sun.jersey.server.wadl;uses:="javax.ws.rs.core,com.sun.research.ws.wadl,javax.xml.namespace,com.sun.jersey.api.model,javax.xml.bind,javax.ws.rs,com.sun.jersey.server.wadl.generators,com.sun.jersey.server.impl.modelapi.annotation,com.sun.jersey.server.impl";version="1.13.0",com.sun.jersey.server.impl.model.method.dispatch;uses:="com.sun.jersey.api.model,com.sun.jersey.api.core,com.sun.jersey.spi.container,com.sun.jersey.server.impl.inject,com.sun.jersey.api,javax.ws.rs.core,com.sun.jersey.core.spi.factory,com.sun.jersey.spi.inject,com.sun.jersey.spi.dispatch,com.sun.jersey.core.spi.component,javax.ws.rs,com.sun.jersey.server.impl.model.parameter.multivalued,com.sun.jersey.api.representation,com.sun.jersey.api.container";version="1.13.0",com.sun.jersey.server.impl;uses:="javax.naming,com.sun.jersey.api.core,com.sun.jersey.core.header,javax.ws.rs.core,com.sun.jersey.server.impl.model,com.sun.jersey.spi.container";version="1.13.0",com.sun.jersey.server.wadl.generators.resourcedoc;uses:="com.sun.jersey.api.model,com.sun.jersey.server.wadl.generators.resourcedoc.model,com.sun.jersey.server.wadl.generators.resourcedoc.xhtml,com.sun.research.ws.wadl,javax.xml.namespace,com.sun.jersey.server.wadl,javax.xml.bind,javax.ws.rs.core";version="1.13.0",com.sun.jersey.server.impl.container.httpserver;uses:="com.sun.net.httpserver,com.sun.jersey.spi.container,javax.ws.rs.core,com.sun.jersey.core.header,com.sun.jersey.api.container,com.sun.jersey.core.util,com.sun.jersey.api.core";version="1.13.0",com.sun.jersey.server.impl.container.filter;uses:="com.sun.jersey.api.model,com.sun.jersey.spi.container,com.sun.jersey.core.spi.component,com.sun.jersey.api.core,javax.ws.rs,com.sun.jersey.server.impl.uri,javax.ws.rs.core";version="1.13.0",com.sun.jersey.server.wadl.generators.resourcedoc.xhtml;uses:="javax.xml.bind,javax.xml.namespace,javax.xml.bind.annotation";version="1.13.0",com.sun.jersey.server.impl.uri.rules;uses:="com.sun.jersey.spi.uri.rules,com.sun.jersey.api.uri,com.sun.jersey.api.core,com.sun.jersey.server.impl.model.method,com.sun.jersey.spi.dispatch,com.sun.jersey.core.header,javax.ws.rs.core,com.sun.jersey.api.model,com.sun.jersey.server.probes,com.sun.jersey.core.reflection,com.sun.jersey.server.impl.template,com.sun.jersey.spi.monitoring,com.sun.jersey.api,com.sun.jersey.spi.container,com.sun.jersey.server.impl.uri,javax.ws.rs,com.sun.jersey.api.container,com.sun.jersey.server.impl.inject,com.sun.jersey.spi.inject,com.sun.jersey.server.impl.uri.rules.automata";version="1.13.0",com.sun.jersey.server.spi.component;uses:="com.sun.jersey.spi.inject,com.sun.jersey.api.model,com.sun.jersey.core.spi.component,com.sun.jersey.api.core,com.sun.jersey.server.impl.inject,com.sun.jersey.api.container,com.sun.jersey.core.spi.component.ioc";version="1.13.0",com.sun.jersey.server.probes;version="1.13.0",com.sun.jersey.server.wadl.generators;uses:="com.sun.research.ws.wadl,javax.xml.bind.annotation,com.sun.jersey.api.model,com.sun.jersey.server.wadl,javax.xml.bind,javax.ws.rs.core,com.sun.jersey.api,javax.xml.namespace,javax.xml.transform,javax.xml.transform.stream";version="1.13.0",com.sun.jersey.server.impl.modelapi.validation;uses:="com.sun.jersey.api.model,javax.ws.rs,com.sun.jersey.impl,com.sun.jersey.api.core,com.sun.jersey.core.reflection,javax.ws.rs.core";version="1.13.0",com.sun.jersey.server.impl.model.method;uses:="com.sun.jersey.api.container,com.sun.jersey.spi.dispatch,com.sun.jersey.api.uri,com.sun.jersey.api.model,com.sun.jersey.server.impl.container.filter,com.sun.jersey.impl,com.sun.jersey.spi.container,com.sun.jersey.spi.inject,com.sun.jersey.api.core,javax.ws.rs.core,com.sun.jersey.core.header";version="1.13.0",com.sun.jersey.server.impl.model;uses:="javax.ws.rs,com.sun.jersey.impl,com.sun.jersey.api.container,com.sun.jersey.core.header,com.sun.jersey.core.header.reader,com.sun.jersey.api.core,javax.ws.rs.core,com.sun.jersey.server.impl.model.method,com.sun.jersey.server.impl.container.filter,com.sun.jersey.api.model,com.sun.jersey.server.impl.wadl,com.sun.jersey.spi.monitoring,com.sun.jersey.server.impl.uri,com.sun.jersey.spi.container,com.sun.jersey.server.impl.inject,com.sun.jersey.spi.inject,com.sun.jersey.api.uri,com.sun.jersey.core.spi.component,com.sun.jersey.server.impl.uri.rules,com.sun.jersey.server.impl.template,com.sun.jersey.api.view,com.sun.jersey.spi.uri.rules";version="1.13.0",com.sun.jersey.server.impl.uri.rules.automata;uses:="com.sun.jersey.server.impl.uri,com.sun.jersey.spi.uri.rules,com.sun.jersey.server.impl.uri.rules,com.sun.jersey.api.uri";version="1.13.0",com.sun.jersey.server.impl.uri;uses:="com.sun.jersey.api.uri,javax.ws.rs.core";version="1.13.0",com.sun.jersey.server.impl.inject;uses:="com.sun.jersey.api.core,com.sun.jersey.spi.inject,javax.ws.rs,com.sun.jersey.api.container,com.sun.jersey.api.model,com.sun.jersey.core.spi.component,com.sun.jersey.core.spi.factory";version="1.13.0",com.sun.jersey.spi.scanning;uses:="org.objectweb.asm,com.sun.jersey.core.reflection,com.sun.jersey.core.spi.scanning,javax.ws.rs,javax.ws.rs.ext";version="1.13.0",com.sun.jersey.spi.resource;uses:="com.sun.jersey.server.impl.resource,com.sun.jersey.server.spi.component";version="1.13.0",com.sun.jersey.spi.template;uses:="com.sun.jersey.api.view,javax.ws.rs.core,com.sun.jersey.api.container";version="1.13.0",com.sun.jersey.spi.dispatch;uses:="com.sun.jersey.api.core";version="1.13.0",com.sun.jersey.spi.uri.rules;uses:="com.sun.jersey.api.core,com.sun.jersey.api.model,com.sun.jersey.spi.container,com.sun.jersey.api.uri";version="1.13.0",com.sun.jersey.spi.container;uses:="javax.ws.rs,com.sun.jersey.api.representation,com.sun.jersey.core.header,com.sun.jersey.spi,javax.ws.rs.core,com.sun.jersey.api.container,com.sun.jersey.api.core,com.sun.jersey.core.util,com.sun.jersey.core.header.reader,com.sun.jersey.server.impl,com.sun.jersey.core.reflection,javax.ws.rs.ext,com.sun.jersey.server.impl.model,com.sun.jersey.api,com.sun.jersey.api.uri,com.sun.jersey.core.spi.factory,com.sun.jersey.spi.monitoring,com.sun.jersey.api.model,com.sun.jersey.core.spi.component,com.sun.jersey.server.impl.application,com.sun.jersey.impl,com.sun.jersey.spi.inject,com.sun.jersey.spi.dispatch,com.sun.jersey.server.impl.inject,com.sun.jersey.core.spi.component.ioc,com.sun.jersey.spi.service";version="1.13.0",com.sun.jersey.spi.monitoring;uses:="com.sun.jersey.api.model,com.sun.jersey.spi.container,javax.ws.rs.ext";version="1.13.0",com.sun.jersey.api;uses:="javax.ws.rs,javax.ws.rs.core,com.sun.jersey.core.header,com.sun.jersey.core.spi.factory";version="1.13.0",com.sun.jersey.api.core;uses:="javax.ws.rs.core,com.sun.jersey.core.spi.scanning,com.sun.jersey.api.model,com.sun.jersey.api.uri,javax.ws.rs,com.sun.jersey.core.header,com.sun.jersey.api.representation,com.sun.jersey.core.util,javax.ws.rs.ext,com.sun.jersey.api.container,com.sun.jersey.spi.scanning,com.sun.jersey.spi.container,com.sun.jersey.server.impl.application";version="1.13.0",com.sun.jersey.api.wadl.config;uses:="com.sun.jersey.server.wadl,com.sun.jersey.api.core,com.sun.jersey.core.reflection,com.sun.jersey.server.wadl.generators";version="1.13.0",com.sun.jersey.api.model;uses:="javax.ws.rs.core,com.sun.jersey.spi.container";version="1.13.0",com.sun.jersey.api.view;version="1.13.0",com.sun.jersey.api.container.filter;uses:="javax.ws.rs,com.sun.jersey.spi.container,javax.ws.rs.core,com.sun.jersey.api.container,com.sun.jersey.api.core,com.sun.jersey.core.util,com.sun.jersey.core.header,com.sun.jersey.api.representation,com.sun.jersey.api.model,javax.annotation.security";version="1.13.0",com.sun.jersey.api.container;uses:="com.sun.jersey.api.core,com.sun.jersey.spi.container,com.sun.jersey.spi.service,com.sun.jersey.core.spi.component.ioc";version="1.13.0",com.sun.jersey.api.container.httpserver;uses:="com.sun.net.httpserver,com.sun.jersey.api.core,com.sun.jersey.api.container,com.sun.jersey.core.spi.component.ioc";version="1.13.0",com.sun.research.ws.wadl;uses:="javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace";version="1.13.0""" - ExportPackageParser.parseAll(export) //throws StackOverflow exception on scala library 2.10.2 - } -} diff --git a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/osgi/ImportPackageTest.scala b/bundle-plugin/src/test/scala/com/yahoo/container/plugin/osgi/ImportPackageTest.scala deleted file mode 100644 index 4011f170da9..00000000000 --- a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/osgi/ImportPackageTest.scala +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.osgi - -import org.scalatest.junit.{AssertionsForJUnit, JUnitSuite} -import org.junit.Test -import ImportPackages.Import -import ExportPackages.{Export, Parameter} -import org.scalatest.Matchers - -/** - * @author tonytv - */ -class ImportPackageTest extends JUnitSuite with AssertionsForJUnit with Matchers { - val referencedPackages = Set("com.yahoo.exported") - val exports = exportByPackageName(Export(List("com.yahoo.exported"), List())) - val exportsWithVersion = exportByPackageName(exports.head._2.copy(parameters = List(Parameter("version", "1.3")))) - - def exportByPackageName(export : Export) = ExportPackages.exportsByPackageName(List(export)) - @Test - def require_that_non_implemented_import_with_matching_export_is_included() { - val imports = calculateImports(referencedPackages, implementedPackages = Set(), exportedPackages = exports) - imports should be (Set(Import("com.yahoo.exported", None))) - } - - - @Test - def require_that_non_implemented_import_without_matching_export_is_excluded() { - val imports = calculateImports(referencedPackages, implementedPackages = Set(), exportedPackages = Map()) - imports should be (Set()) - } - - @Test - def require_that_implemented_import_with_matching_export_is_excluded() { - val imports = calculateImports( - referencedPackages, - implementedPackages = referencedPackages, - exportedPackages = exports) - - imports should be (Set()) - } - - @Test - def require_that_version_is_included() { - val imports = calculateImports(referencedPackages, implementedPackages = Set(), exportedPackages = exportsWithVersion) - - imports should be (Set(Import("com.yahoo.exported", Some("1.3")))) - } - - @Test - def require_that_all_versions_up_to_the_next_major_version_is_in_range() { - Import("foo", Some("1.2")).importVersionRange should be (Some("[1.2,2)")) - } - - // TODO: Detecting guava packages should be based on bundle-symbolicName, not package name. - @Test - def require_that_for_guava_all_future_major_versions_are_in_range() { - val rangeWithInfiniteUpperLimit = Some("[18.1," + ImportPackages.InfiniteVersion + ")") - Import("com.google.common", Some("18.1")).importVersionRange should be (rangeWithInfiniteUpperLimit) - Import("com.google.common.foo", Some("18.1")).importVersionRange should be (rangeWithInfiniteUpperLimit) - - Import("com.google.commonality", Some("18.1")).importVersionRange should be (Some("[18.1,19)")) - } - - @Test - def require_that_none_version_gives_non_version_range() { - Import("foo", None).importVersionRange should be (None) - } - - @Test - def require_that_exception_is_thrown_when_major_component_is_non_numeric() { - intercept[IllegalArgumentException](Import("foo", Some("1notValid.2"))) - } - - @Test - def require_that_osgi_import_supports_missing_version() { - Import("com.yahoo.exported", None).asOsgiImport should be ("com.yahoo.exported") - } - - @Test - def require_that_osgi_import_version_range_includes_all_versions_from_the_current_up_to_the_next_major_version() { - Import("com.yahoo.exported", Some("1.2")).asOsgiImport should be ("com.yahoo.exported;version=\"[1.2,2)\"") - } - - @Test - def require_that_osgi_import_version_range_ignores_qualifier() { - Import("com.yahoo.exported", Some("1.2.3.qualifier")).asOsgiImport should be ("com.yahoo.exported;version=\"[1.2.3,2)\"") - } - - - def calculateImports(referencedPackages : Set[String], - implementedPackages : Set[String], - exportedPackages : Map[String, Export]) : Set[Import] = { - ImportPackages.calculateImports(referencedPackages, implementedPackages, exportedPackages).values.toSet - } -} diff --git a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/util/IOTest.scala b/bundle-plugin/src/test/scala/com/yahoo/container/plugin/util/IOTest.scala deleted file mode 100644 index 7efb0392f5e..00000000000 --- a/bundle-plugin/src/test/scala/com/yahoo/container/plugin/util/IOTest.scala +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.util - -import org.scalatest.junit.{AssertionsForJUnit, JUnitSuite} -import org.junit.Test -import IO.using -import java.io.Closeable - -import org.scalatest.Matchers - -/** - * @author tonytv - */ -class IOTest extends JUnitSuite with AssertionsForJUnit with Matchers { - class ClosingException extends RuntimeException - class FunctionException extends RuntimeException - - object throwWhenClosingResource extends Closeable { - def close() { - throw new ClosingException() - } - } - - def throwFunction(r : throwWhenClosingResource.type) = throw new FunctionException - def nonThrowingFunction(r : throwWhenClosingResource.type) = 42 - - @Test - def require_that_function_exception_is_prioritized_over_closing_exception() { - intercept[FunctionException]{ - using(throwWhenClosingResource, readOnly = false)(throwFunction) - } - } - - @Test - def require_that_closing_exception_is_ignored_when_read_only() { - using(throwWhenClosingResource, readOnly = true)(nonThrowingFunction) should be (nonThrowingFunction(null)) - } - - @Test - def require_that_closing_exception_is_rethrown_when_not_read_only() { - intercept[ClosingException] { - using(throwWhenClosingResource, readOnly = false)(nonThrowingFunction) - } - } -} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index c2956fe738c..6b40577b1b0 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -746,7 +746,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd taskCompletion.getTask().notifyCompleted(); taskCompletionQueue.remove(); } else if (taskCompletion.getDeadlineTimePointMs() <= now) { - log.fine(() -> String.format("Deferred task of type '%s' has exceeded wait deadline; completing with failure", + log.log(LogLevel.WARNING, () -> String.format("Deferred task of type '%s' has exceeded wait deadline; completing with failure", taskCompletion.getTask().getClass().getName())); taskCompletion.getTask().handleFailure(RemoteClusterControllerTask.FailureCondition.DEADLINE_EXCEEDED); taskCompletion.getTask().notifyCompleted(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Request.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Request.java index 5ac15e75127..4ef62ad3fdf 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Request.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Request.java @@ -17,8 +17,8 @@ public abstract class Request<Result> extends RemoteClusterControllerTask { // TODO a lot of this logic could be replaced with a CompleteableFuture private Exception failure = null; - private boolean resultSet = false; - private Result result = null; + protected boolean resultSet = false; + protected Result result = null; private final MasterState masterState; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java index ada068808f7..849d8cc6e7b 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java @@ -32,26 +32,31 @@ public class SetNodeStateRequest extends Request<SetResponse> { private final Map<String, UnitState> newStates; private final SetUnitStateRequest.Condition condition; private final SetUnitStateRequest.ResponseWait responseWait; - + private final WantedStateSetter wantedState; public SetNodeStateRequest(Id.Node id, SetUnitStateRequest setUnitStateRequest) { + this(id, setUnitStateRequest, SetNodeStateRequest::setWantedState); + } + + /** Public for tests. */ + public SetNodeStateRequest(Id.Node id, SetUnitStateRequest setUnitStateRequest, WantedStateSetter wantedState) { super(MasterState.MUST_BE_MASTER); this.id = id; this.newStates = setUnitStateRequest.getNewState(); this.condition = setUnitStateRequest.getCondition(); this.responseWait = setUnitStateRequest.getResponseWait(); + this.wantedState = wantedState; } @Override public SetResponse calculateResult(RemoteClusterControllerTask.Context context) throws StateRestApiException { - SetResponse setResponse = setWantedState( + return wantedState.set( context.cluster, condition, newStates, id.getNode(), context.nodeStateOrHostInfoChangeHandler, context.currentConsolidatedState); - return setResponse; } static NodeState getRequestedNodeState(Map<String, UnitState> newStates, Node n) throws StateRestApiException { @@ -73,6 +78,12 @@ public class SetNodeStateRequest extends Request<SetResponse> { return (responseWait == SetUnitStateRequest.ResponseWait.WAIT_UNTIL_CLUSTER_ACKED); } + @Override + public boolean isFailed() { + // Failure to set a node state is propagated as a 200 with wasModified false. + return super.isFailed() || (resultSet && !result.getWasModified()); + } + static SetResponse setWantedState( ContentCluster cluster, SetUnitStateRequest.Condition condition, diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java index 3f9e2b48eb5..1246ba62313 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java @@ -2,7 +2,8 @@ package com.yahoo.vespa.clustercontroller.core.restapiv2.requests; import com.yahoo.vdslib.distribution.ConfiguredNode; -import com.yahoo.vdslib.state.*; +import com.yahoo.vdslib.state.Node; +import com.yahoo.vdslib.state.NodeType; import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTask; import com.yahoo.vespa.clustercontroller.core.restapiv2.Id; import com.yahoo.vespa.clustercontroller.core.restapiv2.Request; @@ -77,4 +78,10 @@ public class SetNodeStatesForClusterRequest extends Request<SetResponse> { // 'true' here means the current state now equals the request's wanted state. return new SetResponse("ok", true); } + + @Override + public boolean isFailed() { + // Failure to set a node state is propagated as a 200 with wasModified false. + return super.isFailed() || (resultSet && !result.getWasModified()); + } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/WantedStateSetter.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/WantedStateSetter.java new file mode 100644 index 00000000000..6fa7d536c67 --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/WantedStateSetter.java @@ -0,0 +1,26 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core.restapiv2.requests; + +import com.yahoo.vdslib.state.ClusterState; +import com.yahoo.vdslib.state.Node; +import com.yahoo.vespa.clustercontroller.core.ContentCluster; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException; +import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest; +import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.SetResponse; +import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitState; + +import java.util.Map; + +/** + * @author hakon + */ +@FunctionalInterface +public interface WantedStateSetter { + SetResponse set(ContentCluster cluster, + SetUnitStateRequest.Condition condition, + Map<String, UnitState> newStates, + Node node, + NodeStateOrHostInfoChangeHandler stateListener, + ClusterState currentClusterState) throws StateRestApiException; +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java index 4fb244666a4..3f977273054 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java @@ -2,12 +2,15 @@ package com.yahoo.vespa.clustercontroller.core.restapiv2; import com.yahoo.vdslib.state.NodeType; +import com.yahoo.vespa.clustercontroller.core.MasterInterface; import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTask; import com.yahoo.vespa.clustercontroller.core.restapiv2.requests.SetNodeStateRequest; +import com.yahoo.vespa.clustercontroller.core.restapiv2.requests.WantedStateSetter; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.DeadlineExceededException; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.InvalidContentException; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.MissingUnitException; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.OperationNotSupportedForUnitException; +import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.UnknownMasterException; import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest; import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.SetResponse; @@ -26,6 +29,9 @@ import static org.hamcrest.core.StringContains.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class SetNodeStateTest extends StateRestApiTest { @@ -426,4 +432,30 @@ public class SetNodeStateTest extends StateRestApiTest { request.getResult(); } + @Test + public void no_fail_if_modified() throws StateRestApiException { + assertFalse(isFailed(true)); + } + + @Test + public void fail_if_not_modified() throws StateRestApiException { + assertTrue(isFailed(false)); + } + + private boolean isFailed(boolean wasModified) throws StateRestApiException { + WantedStateSetter wantedStateSetter = mock(WantedStateSetter.class); + SetNodeStateRequest request = new SetNodeStateRequest( + createDummyId(), + new SetUnitStateRequestImpl("music/storage/1").setNewState("user", "maintenance", "whatever reason."), + wantedStateSetter); + SetResponse response = new SetResponse("some reason", wasModified); + when(wantedStateSetter.set(any(), any(), any(), any(), any(), any())).thenReturn(response); + + RemoteClusterControllerTask.Context context = mock(RemoteClusterControllerTask.Context.class); + MasterInterface masterInterface = mock(MasterInterface.class); + context.masterInfo = masterInterface; + when(masterInterface.isMaster()).thenReturn(true); + request.doRemoteFleetControllerTask(context); + return request.isFailed(); + } } diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java index 83aca36b8b1..af7f5776f74 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java @@ -4,8 +4,7 @@ package com.yahoo.config.model; /** * Interface for factories of config models. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public interface ConfigModelInstanceFactory<MODEL extends ConfigModel> { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 2005f49213e..a69c524e863 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -694,19 +694,14 @@ public class RankProfile implements Serializable, Cloneable { private void compileThis(QueryProfileRegistry queryProfiles) { parseExpressions(); - checkNameCollisions(getMacros(), getConstants()); - Map<String, Macro> compiledMacros = new LinkedHashMap<>(); - for (Map.Entry<String, Macro> macroEntry : getMacros().entrySet()) { - Macro compiledMacro = macroEntry.getValue().clone(); - compiledMacro.setRankingExpression(compile(macroEntry.getValue().getRankingExpression(), - queryProfiles, - getConstants(), Collections.<String, Macro>emptyMap())); - compiledMacros.put(macroEntry.getKey(), compiledMacro); - } - macros = compiledMacros; - Map<String, Macro> inlineMacros = keepInline(compiledMacros); + // Macro compiling first pass: compile inline macros without resolving other macros + Map<String, Macro> inlineMacros = compileMacros(getInlineMacros(), queryProfiles, Collections.emptyMap()); + + // Macro compiling second pass: compile all macros and insert previously compiled inline macros + macros = compileMacros(getMacros(), queryProfiles, inlineMacros); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, getConstants(), inlineMacros); secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, getConstants(), inlineMacros); } @@ -719,12 +714,22 @@ public class RankProfile implements Serializable, Cloneable { } } - private Map<String, Macro> keepInline(Map<String, Macro> macros) { - Map<String, Macro> inlineMacros = new HashMap<>(); - for (Map.Entry<String, Macro> entry : macros.entrySet()) - if (entry.getValue().getInline()) - inlineMacros.put(entry.getKey(), entry.getValue()); - return inlineMacros; + private Map<String, Macro> getInlineMacros() { + return getMacros().entrySet().stream().filter(x -> x.getValue().getInline()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private Map<String, Macro> compileMacros(Map<String, Macro> macros, + QueryProfileRegistry queryProfiles, + Map<String, Macro> inlineMacros) { + Map<String, Macro> compiledMacros = new LinkedHashMap<>(); + for (Map.Entry<String, Macro> entry : macros.entrySet()) { + Macro macro = entry.getValue().clone(); + RankingExpression exp = compile(macro.getRankingExpression(), queryProfiles, getConstants(), inlineMacros); + macro.setRankingExpression(exp); + compiledMacros.put(entry.getKey(), macro); + } + return compiledMacros; } private RankingExpression compile(RankingExpression expression, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 0f1918f99d6..55f3a94bb70 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -8,6 +8,7 @@ import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Ranking; +import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.Sorting; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; @@ -18,10 +19,12 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isSupportedComplexField; /** * The set of all attribute fields defined by a search definition @@ -45,14 +48,14 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce /** Derives everything from a field */ @Override protected void derive(ImmutableSDField field, Search search) { - if (unsupportedFieldType(field)) { + if (unsupportedFieldType(field, search.getDocument())) { return; // Ignore complex struct and map fields for indexed search (only supported for streaming search) } if (field.isImportedField()) { deriveImportedAttributes(field); - } else if (isArrayOfSimpleStruct(field)) { + } else if (isArrayOfSimpleStruct(field, search.getDocument())) { deriveArrayOfSimpleStruct(field); - } else if (isMapOfSimpleStruct(field)) { + } else if (isMapOfSimpleStruct(field, search.getDocument())) { deriveMapOfSimpleStruct(field); } else if (isMapOfPrimitiveType(field)) { deriveMapOfPrimitiveType(field); @@ -61,11 +64,9 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } } - private static boolean unsupportedFieldType(ImmutableSDField field) { + private static boolean unsupportedFieldType(ImmutableSDField field, SDDocumentType docType) { return (field.usesStructOrMap() && - !isArrayOfSimpleStruct(field) && - !isMapOfSimpleStruct(field) && - !isMapOfPrimitiveType(field) && + !isSupportedComplexField(field, docType) && !field.getDataType().equals(PositionDataType.INSTANCE) && !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); } @@ -153,6 +154,13 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce return Collections.unmodifiableCollection(attributes.values()); } + public Collection<Attribute> structFieldAttributes(String baseFieldName) { + String structPrefix = baseFieldName + "."; + return attributes().stream() + .filter(attribute -> attribute.getName().startsWith(structPrefix)) + .collect(Collectors.toList()); + } + public String toString() { return "attributes " + getName(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java index 89d72756512..72eb1c96e0f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java @@ -1,3 +1,4 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.document; import com.yahoo.document.ArrayDataType; @@ -6,6 +7,11 @@ import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; +import com.yahoo.document.TemporaryStructuredDataType; + +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; /** * Utils used to check whether a complex field supports being represented as struct field attributes. @@ -13,34 +19,64 @@ import com.yahoo.document.StructDataType; * Currently we support: * - array of simple struct * - map of primitive type to simple struct + * - map of primitive type to primitive type * * @author geirst */ public class ComplexAttributeFieldUtils { - public static boolean isArrayOfSimpleStruct(ImmutableSDField field) { - DataType fieldType = field.getDataType(); + public static boolean isSupportedComplexField(ImmutableSDField field, SDDocumentType docType) { + return (isArrayOfSimpleStruct(field, docType) || + isMapOfSimpleStruct(field, docType) || + isMapOfPrimitiveType(field)); + } + + public static boolean isSupportedComplexField(DataType fieldType) { + return (isArrayOfSimpleStruct(fieldType) || + isMapOfSimpleStruct(fieldType) || + isMapOfPrimitiveType(fieldType)); + } + + public static boolean isArrayOfSimpleStruct(ImmutableSDField field, SDDocumentType docType) { + return isArrayOfSimpleStruct(field.getDataType(), Optional.of(docType)); + } + + public static boolean isArrayOfSimpleStruct(DataType fieldType) { + return isArrayOfSimpleStruct(fieldType, Optional.empty()); + } + + private static boolean isArrayOfSimpleStruct(DataType fieldType, Optional<SDDocumentType> docType) { if (fieldType instanceof ArrayDataType) { ArrayDataType arrayType = (ArrayDataType)fieldType; - return isSimpleStruct(arrayType.getNestedType()); + return isSimpleStruct(arrayType.getNestedType(), docType); } else { return false; } } - public static boolean isMapOfSimpleStruct(ImmutableSDField field) { - DataType fieldType = field.getDataType(); + public static boolean isMapOfSimpleStruct(ImmutableSDField field, SDDocumentType docType) { + return isMapOfSimpleStruct(field.getDataType(), Optional.of(docType)); + } + + public static boolean isMapOfSimpleStruct(DataType fieldType) { + return isMapOfSimpleStruct(fieldType, Optional.empty()); + } + + private static boolean isMapOfSimpleStruct(DataType fieldType, Optional<SDDocumentType> docType) { if (fieldType instanceof MapDataType) { MapDataType mapType = (MapDataType)fieldType; return isPrimitiveType(mapType.getKeyType()) && - isSimpleStruct(mapType.getValueType()); + isSimpleStruct(mapType.getValueType(), docType); } else { return false; } } public static boolean isMapOfPrimitiveType(ImmutableSDField field) { - DataType fieldType = field.getDataType(); + return isMapOfPrimitiveType(field.getDataType()); + } + + public static boolean isMapOfPrimitiveType(DataType fieldType) { if (fieldType instanceof MapDataType) { MapDataType mapType = (MapDataType)fieldType; return isPrimitiveType(mapType.getKeyType()) && @@ -50,11 +86,15 @@ public class ComplexAttributeFieldUtils { } } - private static boolean isSimpleStruct(DataType type) { + private static boolean isSimpleStruct(DataType type, Optional<SDDocumentType> docType) { if (type instanceof StructDataType && !(type.equals(PositionDataType.INSTANCE))) { StructDataType structType = (StructDataType) type; - for (Field innerField : structType.getFields()) { + Collection<Field> structFields = getStructFields(structType, docType); + if (structFields.isEmpty()) { + return false; + } + for (Field innerField : structFields) { if (!isPrimitiveType(innerField.getDataType())) { return false; } @@ -65,6 +105,19 @@ public class ComplexAttributeFieldUtils { } } + private static Collection<Field> getStructFields(StructDataType structType, Optional<SDDocumentType> docType) { + // The struct data type might be unresolved at this point. If so we use the document type to resolve it. + if (docType.isPresent() && (structType instanceof TemporaryStructuredDataType)) { + SDDocumentType realStructType = docType.get().getOwnedType(structType.getName()); + if (structType != null) { + return realStructType.getDocumentType().getFields(); + } + return Collections.emptyList(); + } else { + return structType.getFields(); + } + } + private static boolean isPrimitiveType(DataType dataType) { return dataType.equals(DataType.BYTE) || dataType.equals(DataType.INT) || @@ -74,10 +127,10 @@ public class ComplexAttributeFieldUtils { dataType.equals(DataType.STRING); } - public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field) { - if (isArrayOfSimpleStruct(field)) { + public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field, SDDocumentType docType) { + if (isArrayOfSimpleStruct(field, docType)) { return hasOnlyStructFieldAttributes(field); - } else if (isMapOfSimpleStruct(field)) { + } else if (isMapOfSimpleStruct(field, docType)) { return hasSingleAttribute(field.getStructField("key")) && hasOnlyStructFieldAttributes(field.getStructField("value")); } else if (isMapOfPrimitiveType(field)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java index 28abbe43968..46ac3cc1691 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java @@ -122,10 +122,16 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain } public void apply(SDField field) { - Attribute attribute = field.getAttributes().get(name); + Attribute attribute = null; + if (attributeIsSuffixOfStructField(field.getName())) { + attribute = field.getAttributes().get(field.getName()); + } if (attribute == null) { - attribute = new Attribute(name, field.getDataType()); - field.addAttribute(attribute); + attribute = field.getAttributes().get(name); + if (attribute == null) { + attribute = new Attribute(name, field.getDataType()); + field.addAttribute(attribute); + } } if (huge != null) { @@ -154,4 +160,8 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain } } + private boolean attributeIsSuffixOfStructField(String fieldName) { + return ((fieldName.indexOf('.') != -1) && fieldName.endsWith(name)); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index b51524b7e62..2d2fc10d9c5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -83,7 +83,7 @@ public class ImplicitSummaries extends Processor { } } - if (addedSummaryField != null && isComplexFieldWithOnlyStructFieldAttributes(field)) { + if (addedSummaryField != null && isComplexFieldWithOnlyStructFieldAttributes(field, search.getDocument())) { addedSummaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java new file mode 100644 index 00000000000..005c1dd8b2e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -0,0 +1,85 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; +import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.SearchCluster; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Validates that complex fields (of type struct or map) that have struct field attributes are supported. + * + * Only applies for indexed search clusters. + * + * @author geirst + */ +public class ComplexAttributeFieldsValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + List<AbstractSearchCluster> searchClusters = model.getSearchClusters(); + for (AbstractSearchCluster cluster : searchClusters) { + if (cluster.isStreaming()) { + continue; + } + SearchCluster searchCluster = (SearchCluster) cluster; + for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) { + validateComplexFields(searchCluster.getClusterName(), spec.getSearchDefinition().getSearch()); + } + } + } + + private static void validateComplexFields(String clusterName, Search search) { + String unsupportedFields = search.allFields() + .filter(field -> isUnsupportedComplexField(field)) + .map(ComplexAttributeFieldsValidator::toString) + .collect(Collectors.joining(", ")); + + if (!unsupportedFields.isEmpty()) { + throw new IllegalArgumentException( + String.format("For cluster '%s', search '%s': The following complex fields do not support using struct field attributes: %s. " + + "Only supported for the following complex field types: array or map of struct with primitive types, map of primitive types", + clusterName, search.getName(), unsupportedFields)); + } + } + + private static boolean isUnsupportedComplexField(ImmutableSDField field) { + return (field.usesStructOrMap() && + !isSupportedComplexField(field) && + hasStructFieldAttributes(field.getStructFields())); + } + + private static boolean isSupportedComplexField(ImmutableSDField field) { + return (ComplexAttributeFieldUtils.isSupportedComplexField(field.getDataType()) || + field.getDataType().equals(PositionDataType.INSTANCE) || + field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); + } + + private static String toString(ImmutableSDField field) { + return field.getName() + " (" + StringUtils.join(getStructFieldAttributes(field.getStructFields()), ", ") + ")"; + } + + private static boolean hasStructFieldAttributes(Collection<? extends ImmutableSDField> structFields) { + return !getStructFieldAttributes(structFields).isEmpty(); + } + + private static List<String> getStructFieldAttributes(Collection<? extends ImmutableSDField> structFields) { + List<String> result = new ArrayList<>(); + for (ImmutableSDField structField : structFields) { + structField.getAttributes().values().forEach(attr -> result.add(attr.getName())); + result.addAll(getStructFieldAttributes(structField.getStructFields())); + } + return result; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 523ced52306..c08e81b250f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -52,6 +52,7 @@ public class Validation { } new ComponentValidator().validate(model, deployState); new SearchDataTypeValidator().validate(model, deployState); + new ComplexAttributeFieldsValidator().validate(model, deployState); new StreamingValidator().validate(model, deployState); new RankSetupValidator(force).validate(model, deployState); new NoPrefixForIndexes().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java index bd287f83a1a..deae0a89c56 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java @@ -36,6 +36,7 @@ public class DocumentDatabaseChangeValidator { public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { List<VespaConfigChangeAction> result = new ArrayList<>(); result.addAll(validateAttributeChanges(overrides, now)); + result.addAll(validateStructFieldAttributeChanges(overrides, now)); result.addAll(validateIndexingScriptChanges(overrides, now)); result.addAll(validateDocumentTypeChanges(overrides, now)); return result; @@ -49,6 +50,11 @@ public class DocumentDatabaseChangeValidator { nextDatabase.getDerivedConfiguration().getIndexSchema(), nextDocType).validate(overrides, now); } + private List<VespaConfigChangeAction> validateStructFieldAttributeChanges(ValidationOverrides overrides, Instant now) { + return new StructFieldAttributeChangeValidator(currentDocType, currentDatabase.getDerivedConfiguration().getAttributeFields(), + nextDocType, nextDatabase.getDerivedConfiguration().getAttributeFields()).validate(overrides, now); + } + private List<VespaConfigChangeAction> validateIndexingScriptChanges(ValidationOverrides overrides, Instant now) { return new IndexingScriptChangeValidator(currentDatabase.getDerivedConfiguration().getSearch(), nextDatabase.getDerivedConfiguration().getSearch()).validate(overrides, now); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java new file mode 100644 index 00000000000..520494f1697 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java @@ -0,0 +1,132 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.StringTokenizer; +import java.util.stream.Collectors; + +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; + +/** + * Validates the changes between the current and next set of struct field attributes in a document database. + + * Complex fields of the following types are considered (as they might have struct field attributes): + * - array of simple struct + * - map of simple struct + * - map of primitive types + * + * @author geirst + */ +public class StructFieldAttributeChangeValidator { + + private final NewDocumentType currentDocType; + private final AttributeFields currentAttributes; + private final NewDocumentType nextDocType; + private final AttributeFields nextAttributes; + + public StructFieldAttributeChangeValidator(NewDocumentType currentDocType, + AttributeFields currentAttributes, + NewDocumentType nextDocType, + AttributeFields nextAttributes) { + this.currentDocType = currentDocType; + this.currentAttributes = currentAttributes; + this.nextDocType = nextDocType; + this.nextAttributes = nextAttributes; + } + + public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { + List<VespaConfigChangeAction> result = new ArrayList(); + for (Field currentField : currentDocType.getAllFields()) { + Field nextField = nextDocType.getField(currentField.getName()); + if (nextField != null) { + result.addAll(validateAddAttributeAspect(new Context(currentField, currentAttributes), + new Context(nextField, nextAttributes), + overrides, now)); + } + } + return result; + } + + private List<VespaConfigChangeAction> validateAddAttributeAspect(Context current, Context next, ValidationOverrides overrides, Instant now) { + return next.structFieldAttributes.stream() + .filter(nextAttr -> current.hasFieldFor(nextAttr) && + !current.hasStructFieldAttribute(nextAttr)) + .map(nextAttr -> VespaRefeedAction.of("field-type-change", + overrides, + new ChangeMessageBuilder(nextAttr.getName()) + .addChange("add attribute aspect").build(), + now)) + .collect(Collectors.toList()); + } + + private static class Context { + public Field field; + public Collection<Attribute> structFieldAttributes; + + public Context(Field field, AttributeFields attributes) { + this.field = field; + this.structFieldAttributes = attributes.structFieldAttributes(field.getName()); + } + + public DataType dataType() { + return field.getDataType(); + } + + public boolean hasStructFieldAttribute(Attribute structFieldAttribute) { + return structFieldAttributes.stream() + .anyMatch(attr -> attr.getName().equals(structFieldAttribute.getName())); + } + + public boolean hasFieldFor(Attribute structFieldAttribute) { + StringTokenizer fieldNames = new StringTokenizer(structFieldAttribute.getName(), "."); + if (!fieldNames.nextToken().equals(field.getName())) { + return false; + } + if (isArrayOfSimpleStruct(dataType())) { + StructDataType nestedType = (StructDataType)((ArrayDataType)dataType()).getNestedType(); + if (hasLastFieldInStructType(fieldNames, nestedType)) { + return true; + } + } else if (isMapOfSimpleStruct(dataType())) { + MapDataType mapType = (MapDataType)dataType(); + StructDataType valueType = (StructDataType)mapType.getValueType(); + String subFieldName = fieldNames.nextToken(); + if (subFieldName.equals("key") && !fieldNames.hasMoreTokens()) { + return true; + } else if (subFieldName.equals("value") && hasLastFieldInStructType(fieldNames, valueType)) { + return true; + } + } else if (isMapOfPrimitiveType(dataType())) { + String subFieldName = fieldNames.nextToken(); + if ((subFieldName.equals("key") || subFieldName.equals("value")) && + !fieldNames.hasMoreTokens()) { + return true; + } + } + return false; + } + + private static boolean hasLastFieldInStructType(StringTokenizer fieldNames, StructDataType structType) { + return structType.getField(fieldNames.nextToken()) != null && !fieldNames.hasMoreTokens(); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index b55cfb636ea..9801eab9f2b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -41,6 +41,8 @@ public class ContainerDocumentApi implements FeederConfig.Producer { private void setupHandlers(ContainerCluster cluster) { cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandler", "feed")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemove", "remove")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", "removelocation")); cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerGet", "get")); cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerVisit", "visit")); cluster.addComponent(newVespaClientHandler("com.yahoo.document.restapi.resource.RestApi", "document/v1/*")); diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index c4c1a04da0b..1bf87754ce0 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -979,8 +979,7 @@ String fieldBody(SDField field, Search search, SDDocumentType document) : { } /** * This rule consumes a single element of a struct subfield body block. - * Only elements that make sense for streaming search are allowed, - * since only streaming search supports structs for now. + * Only elements that are supported in streaming search and indexed search (with complex attributes) are allowed. * * @param field The field being built. * @param search The search object to add content to. @@ -991,6 +990,7 @@ String structFieldBody(FieldOperationContainer field, Search search, SDDocumentT { ( summaryInField(field) | indexing(field) | + attribute(field) | match(field) | queryCommand(field) | structField(field, search, document) | diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg index 9e6b5cea55e..d42ed223045 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -3,7 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].removeifzero false attribute[].createifnonexistent false -attribute[].fastsearch false +attribute[].fastsearch true attribute[].huge false attribute[].sortascending true attribute[].sortfunction UCA diff --git a/config-model/src/test/derived/array_of_struct_attribute/test.sd b/config-model/src/test/derived/array_of_struct_attribute/test.sd index 5b2d50cbdba..3678c2942b5 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/test.sd +++ b/config-model/src/test/derived/array_of_struct_attribute/test.sd @@ -8,6 +8,7 @@ search test { indexing: summary struct-field name { indexing: attribute + attribute: fast-search } struct-field weight { indexing: attribute diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg index cdd929a1818..4e0c3d60338 100644 --- a/config-model/src/test/derived/map_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -3,7 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].removeifzero false attribute[].createifnonexistent false -attribute[].fastsearch false +attribute[].fastsearch true attribute[].huge false attribute[].sortascending true attribute[].sortfunction UCA diff --git a/config-model/src/test/derived/map_attribute/test.sd b/config-model/src/test/derived/map_attribute/test.sd index da31f575396..87fc13d48ed 100644 --- a/config-model/src/test/derived/map_attribute/test.sd +++ b/config-model/src/test/derived/map_attribute/test.sd @@ -4,6 +4,7 @@ search test { indexing: summary struct-field key { indexing: attribute + attribute: fast-search } struct-field value { indexing: attribute diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg index 95cf8f761a0..7e38ba06f74 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -3,7 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].removeifzero false attribute[].createifnonexistent false -attribute[].fastsearch false +attribute[].fastsearch true attribute[].huge false attribute[].sortascending true attribute[].sortfunction UCA @@ -83,7 +83,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].removeifzero false attribute[].createifnonexistent false -attribute[].fastsearch false +attribute[].fastsearch true attribute[].huge false attribute[].sortascending true attribute[].sortfunction UCA diff --git a/config-model/src/test/derived/map_of_struct_attribute/test.sd b/config-model/src/test/derived/map_of_struct_attribute/test.sd index e139a7f8168..929fa8dcbbe 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/test.sd +++ b/config-model/src/test/derived/map_of_struct_attribute/test.sd @@ -8,6 +8,7 @@ search test { indexing: summary struct-field key { indexing: attribute + attribute: fast-search } struct-field value.name { indexing: attribute @@ -23,6 +24,7 @@ search test { } struct-field value.name { indexing: attribute + attribute: fast-search } } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index 97e1ab9aeb9..a0dd18aeea9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -8,9 +8,10 @@ import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; -import java.util.List; +import java.util.Optional; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author bratseth @@ -67,6 +68,7 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase assertEquals("7.0 * (9 + attribute(a))", child.getFirstPhaseRanking().getRoot().toString()); } + @Test public void testConstants() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); @@ -122,36 +124,63 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(new QueryProfileRegistry()); assertEquals("17.0", parent.getFirstPhaseRanking().getRoot().toString()); assertEquals("0.0", parent.getSecondPhaseRanking().getRoot().toString()); - List<Pair<String, String>> parentRankProperties = new RawRankProfile(parent, - new QueryProfileRegistry(), - new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(foo).rankingScript,10.0)", - parentRankProperties.get(0).toString()); - assertEquals("(rankingExpression(firstphase).rankingScript,17.0)", - parentRankProperties.get(2).toString()); - assertEquals("(rankingExpression(secondphase).rankingScript,0.0)", - parentRankProperties.get(4).toString()); + assertEquals("10.0", getRankingExpression("foo", parent, s)); + assertEquals("17.0", getRankingExpression("firstphase", parent, s)); + assertEquals("0.0", getRankingExpression("secondphase", parent, s)); RankProfile child = rankProfileRegistry.getRankProfile(s, "child").compile(new QueryProfileRegistry()); assertEquals("31.0 + bar + arg(4.0)", child.getFirstPhaseRanking().getRoot().toString()); assertEquals("24.0", child.getSecondPhaseRanking().getRoot().toString()); - List<Pair<String, String>> childRankProperties = new RawRankProfile(child, - new QueryProfileRegistry(), - new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(foo).rankingScript,12.0)", - childRankProperties.get(0).toString()); - assertEquals("(rankingExpression(bar).rankingScript,14.0)", - childRankProperties.get(1).toString()); - assertEquals("(rankingExpression(boz).rankingScript,3.0)", - childRankProperties.get(2).toString()); - assertEquals("(rankingExpression(baz).rankingScript,9.0 + rankingExpression(boz))", - childRankProperties.get(3).toString()); - assertEquals("(rankingExpression(arg).rankingScript,a1 * 2)", - childRankProperties.get(4).toString()); - assertEquals("(rankingExpression(firstphase).rankingScript,31.0 + rankingExpression(bar) + rankingExpression(arg@))", - censorBindingHash(childRankProperties.get(7).toString())); - assertEquals("(rankingExpression(secondphase).rankingScript,24.0)", - childRankProperties.get(9).toString()); + assertEquals("12.0", getRankingExpression("foo", child, s)); + assertEquals("12.0", getRankingExpression("baz", child, s)); + assertEquals("3.0", getRankingExpression("boz", child, s)); + assertEquals("14.0", getRankingExpression("bar", child, s)); + assertEquals("a1 * 2", getRankingExpression("arg", child, s)); + assertEquals("31.0 + rankingExpression(bar) + rankingExpression(arg@)", getRankingExpression("firstphase", child, s)); + assertEquals("24.0", getRankingExpression("secondphase", child, s)); + } + + @Test + public void testNonTopLevelInlining() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field a type double { \n" + + " indexing: attribute \n" + + " }\n" + + " field b type double { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: A + C + D\n" + + " }\n" + + " macro inline D() {\n" + + " expression: B + 1\n" + + " }\n" + + " macro C() {\n" + + " expression: A + B\n" + + " }\n" + + " macro inline B() {\n" + + " expression: attribute(b)\n" + + " }\n" + + " macro inline A() {\n" + + " expression: attribute(a)\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(); + Search s = builder.getSearch(); + + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); + assertEquals("attribute(a) + C + (attribute(b) + 1)", test.getFirstPhaseRanking().getRoot().toString()); + assertEquals("attribute(a) + attribute(b)", getRankingExpression("C", test, s)); + assertEquals("attribute(b) + 1", getRankingExpression("D", test, s)); } /** @@ -177,4 +206,16 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase return b.toString(); } + private String getRankingExpression(String name, RankProfile rankProfile, Search search) { + Optional<String> rankExpression = + new RawRankProfile(rankProfile, new QueryProfileRegistry(), new AttributeFields(search)) + .configProperties() + .stream() + .filter(r -> r.getFirst().equals("rankingExpression(" + name + ").rankingScript")) + .map(Pair::getSecond) + .findFirst(); + assertTrue(rankExpression.isPresent()); + return censorBindingHash(rankExpression.get()); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java index bd762c64c00..da28dd56694 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -74,8 +74,8 @@ public class AttributeListTestCase extends SearchDefinitionTestCase { Search search = SearchBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); - assertAttribute("elem_array.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); - assertAttribute("elem_array.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("elem_array.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); + assertAttribute("elem_array.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); assertTrue(!attributes.hasNext()); } @@ -84,18 +84,19 @@ public class AttributeListTestCase extends SearchDefinitionTestCase { Search search = SearchBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); - assertAttribute("str_elem_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); - assertAttribute("str_elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); - assertAttribute("str_elem_map.value.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next()); - assertAttribute("int_elem_map.key", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next()); - assertAttribute("int_elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("str_elem_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); + assertAttribute("str_elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, false, attributes.next()); + assertAttribute("str_elem_map.value.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); + assertAttribute("int_elem_map.key", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); + assertAttribute("int_elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); assertTrue(!attributes.hasNext()); } - private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, Attribute attr) { + private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, boolean isFastSearch, Attribute attr) { assertEquals(name, attr.getName()); assertEquals(type, attr.getType()); assertEquals(collection, attr.getCollectionType()); + assertEquals(isFastSearch, attr.isFastSearch()); } @Test @@ -110,7 +111,7 @@ public class AttributeListTestCase extends SearchDefinitionTestCase { "}")).getSearch(); Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); - assertAttribute("pos_array_zcurve", Attribute.Type.LONG, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("pos_array_zcurve", Attribute.Type.LONG, Attribute.CollectionType.ARRAY, true, attributes.next()); assertTrue(!attributes.hasNext()); } @@ -119,9 +120,9 @@ public class AttributeListTestCase extends SearchDefinitionTestCase { Search search = SearchBuilder.buildFromFile("src/test/derived/map_attribute/test.sd"); Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); - assertAttribute("str_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); - assertAttribute("str_map.value", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); - assertAttribute("int_map.key", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("str_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); + assertAttribute("str_map.value", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, false, attributes.next()); + assertAttribute("int_map.key", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); assertTrue(!attributes.hasNext()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java index 4cdc48b330e..91a89c204c9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java @@ -1,3 +1,4 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.document; import com.yahoo.searchdefinition.Search; @@ -6,126 +7,214 @@ import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class ComplexAttributeFieldUtilsTestCase { - private static ImmutableSDField createField(String fieldName, String sdFieldContent) throws ParseException { - String sdContent = joinLines("search test {", - " document test {", - " struct elem {", - " field name type string {}", - " field weight type string {}", - " }", - sdFieldContent, - " }", - "}"); - Search search = SearchBuilder.createFromString(sdContent).getSearch(); - return search.getConcreteField(fieldName); + private static class FixtureBase { + private final Search search; + private final ImmutableSDField field; + + public FixtureBase(String fieldName, String sdContent) throws ParseException { + search = SearchBuilder.createFromString(sdContent).getSearch(); + field = search.getConcreteField(fieldName); + } + + public ImmutableSDField field() { + return field; + } + + public SDDocumentType docType() { + return search.getDocument(); + } + + public boolean isSupportedComplexField() { + return ComplexAttributeFieldUtils.isSupportedComplexField(field(), docType()); + } + + public boolean isArrayOfSimpleStruct() { + return ComplexAttributeFieldUtils.isArrayOfSimpleStruct(field(), docType()); + } + + public boolean isMapOfSimpleStruct() { + return ComplexAttributeFieldUtils.isMapOfSimpleStruct(field(), docType()); + } + + public boolean isMapOfPrimitiveType() { + return ComplexAttributeFieldUtils.isMapOfPrimitiveType(field()); + } + + public boolean isComplexFieldWithOnlyStructFieldAttributes() { + return ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes(field(), docType()); + } + } + + private static class Fixture extends FixtureBase { + + public Fixture(String fieldName, String sdFieldContent) throws ParseException { + super(fieldName, joinLines("search test {", + " document test {", + " struct elem {", + " field name type string {}", + " field weight type string {}", + " }", + sdFieldContent, + " }", + "}")); + } + } + + private static class ComplexFixture extends FixtureBase { + + public ComplexFixture(String fieldName, String sdFieldContent) throws ParseException { + super(fieldName, joinLines("search test {", + " document test {", + " struct elem {", + " field name type string {}", + " field weight type array<string> {}", + " }", + sdFieldContent, + " }", + "}")); + } } @Test public void array_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { - ImmutableSDField field = createField("elem_array", + Fixture f = new Fixture("elem_array", joinLines("field elem_array type array<elem> {", " indexing: summary", " struct-field name { indexing: attribute }", " struct-field weight { indexing: attribute }", "}")); - assertTrue(isArrayOfSimpleStruct(field)); - assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isArrayOfSimpleStruct()); + assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); } @Test public void array_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { - ImmutableSDField field = createField("elem_array", + Fixture f = new Fixture("elem_array", joinLines("field elem_array type array<elem> {", " indexing: summary", " struct-field weight { indexing: attribute }", "}")); - assertTrue(isArrayOfSimpleStruct(field)); - assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isArrayOfSimpleStruct()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); } @Test public void map_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { - ImmutableSDField field = createField("elem_map", + Fixture f = new Fixture("elem_map", joinLines("field elem_map type map<string, elem> {", " indexing: summary", " struct-field key { indexing: attribute }", " struct-field value.name { indexing: attribute }", " struct-field value.weight { indexing: attribute }", "}")); - assertTrue(isMapOfSimpleStruct(field)); - assertFalse(isMapOfPrimitiveType(field)); - assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); } @Test public void map_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { { - ImmutableSDField field = createField("elem_map", + Fixture f = new Fixture("elem_map", joinLines("field elem_map type map<int, elem> {", " indexing: summary", " struct-field value.name { indexing: attribute }", " struct-field value.weight { indexing: attribute }", "}")); - assertTrue(isMapOfSimpleStruct(field)); - assertFalse(isMapOfPrimitiveType(field)); - assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); } { - ImmutableSDField field = createField("elem_map", + Fixture f = new Fixture("elem_map", joinLines("field elem_map type map<int, elem> {", " indexing: summary", " struct-field key { indexing: attribute }", " struct-field value.weight { indexing: attribute }", "}")); - assertTrue(isMapOfSimpleStruct(field)); - assertFalse(isMapOfPrimitiveType(field)); - assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); } } @Test public void map_of_primitive_type_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { - ImmutableSDField field = createField("str_map", - joinLines("field str_map type map<string, string> {", - " indexing: summary", - " struct-field key { indexing: attribute }", - " struct-field value { indexing: attribute }", - "}")); - assertTrue(isMapOfPrimitiveType(field)); - assertFalse(isMapOfSimpleStruct(field)); - assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field)); + Fixture f = new Fixture("str_map", + joinLines("field str_map type map<string, string> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfPrimitiveType()); + assertFalse(f.isMapOfSimpleStruct()); + assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); } @Test public void map_of_primitive_type_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { { - ImmutableSDField field = createField("int_map", + Fixture f = new Fixture("int_map", joinLines("field int_map type map<int, int> {", " indexing: summary", " struct-field key { indexing: attribute }", "}")); - assertTrue(isMapOfPrimitiveType(field)); - assertFalse(isMapOfSimpleStruct(field)); - assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfPrimitiveType()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); } { - ImmutableSDField field = createField("int_map", + Fixture f = new Fixture("int_map", joinLines("field int_map type map<int, int> {", " indexing: summary", " struct-field value { indexing: attribute }", "}")); - assertTrue(isMapOfPrimitiveType(field)); - assertFalse(isMapOfSimpleStruct(field)); - assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfPrimitiveType()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + } + + @Test + public void unsupported_complex_field_is_tagged_as_such() throws ParseException { + { + ComplexFixture f = new ComplexFixture("elem_array", + joinLines("field elem_array type array<elem> {", + " struct-field name { indexing: attribute }", + " struct-field weight { indexing: attribute }", + "}")); + assertFalse(f.isSupportedComplexField()); + assertFalse(f.isArrayOfSimpleStruct()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + { + ComplexFixture f = new ComplexFixture("elem_map", + joinLines("field elem_map type map<int, elem> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value.weight { indexing: attribute }", + "}")); + assertFalse(f.isSupportedComplexField()); + assertFalse(f.isArrayOfSimpleStruct()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); } } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java new file mode 100644 index 00000000000..146369d1620 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java @@ -0,0 +1,68 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static com.yahoo.config.model.test.TestUtil.joinLines; + +/** + * @author geirst + */ +public class ComplexAttributeFieldsValidatorTestCase { + + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void throws_exception_when_unsupported_complex_fields_have_struct_field_attributes() throws IOException, SAXException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For cluster 'mycluster', search 'test': " + + "The following complex fields do not support using struct field attributes: " + + "struct_array (struct_array.s1), struct_map (struct_map.key, struct_map.value.s1). " + + "Only supported for the following complex field types: array or map of struct with primitive types, map of primitive types"); + + createModelAndValidate(joinLines("search test {", + " document test {", + " struct s { field s1 type array<int> {} }", + " field struct_array type array<s> {", + " struct-field s1 { indexing: attribute }", + " }", + " field struct_map type map<string,s> {", + " struct-field key { indexing: attribute }", + " struct-field value.s1 { indexing: attribute }", + " }", + " }", + "}")); + } + + private static void createModelAndValidate(String searchDefinition) throws IOException, SAXException { + DeployState deployState = createDeployState(servicesXml(), searchDefinition); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + Validation.validate(model, false, false, deployState); + } + + private static DeployState createDeployState(String servicesXml, String searchDefinition) { + ApplicationPackage app = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .withSearchDefinition(searchDefinition) + .build(); + return new DeployState.Builder().applicationPackage(app).build(true); + } + + private static String servicesXml() { + return joinLines("<services version='1.0'>", + new ContentClusterBuilder().getXml(), + "</services>"); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java index 9572cfebb2a..65ac6c7625e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java @@ -22,6 +22,10 @@ public class ConfigChangeTestUtils { return new VespaRestartAction(message, services); } + public static VespaConfigChangeAction newRefeedAction(String name, String message) { + return VespaRefeedAction.of(name, ValidationOverrides.empty, message, Instant.now()); + } + public static VespaConfigChangeAction newRefeedAction(String name, ValidationOverrides overrides, String message, Instant now) { return VespaRefeedAction.of(name, overrides, message, now); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java index ba736af2159..339f8514f9f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java @@ -31,14 +31,20 @@ public class DocumentDatabaseChangeValidatorTest { @Test public void requireThatAttributeIndexAndDocumentTypeChangesAreDiscovered() throws Exception { - Fixture f = new Fixture("field f1 type string { indexing: summary } " + + Fixture f = new Fixture("struct s { field s1 type string {} } " + + "field f1 type string { indexing: summary } " + "field f2 type string { indexing: summary } " + - "field f3 type int { indexing: summary }", + "field f3 type int { indexing: summary } " + + "field f4 type array<s> { } ", + "struct s { field s1 type string {} } " + "field f1 type string { indexing: attribute | summary } " + "field f2 type string { indexing: index | summary } " + - "field f3 type string { indexing: summary }"); + "field f3 type string { indexing: summary } " + + "field f4 type array<s> { struct-field s1 { indexing: attribute } }"); f.assertValidation(Arrays.asList( newRestartAction("Field 'f1' changed: add attribute aspect"), + newRefeedAction("field-type-change", + "Field 'f4.s1' changed: add attribute aspect"), newRefeedAction("indexing-change", ValidationOverrides.empty, "Field 'f2' changed: add index aspect, indexing script: '{ input f2 | summary f2; }' -> " + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java new file mode 100644 index 00000000000..c224e801fa3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java @@ -0,0 +1,146 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import org.junit.Test; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction; + +/** + * @author geirst + */ +public class StructFieldAttributeChangeValidatorTestCase { + + private static class Fixture extends ContentClusterFixture { + private StructFieldAttributeChangeValidator structFieldAttributeValidator; + private DocumentTypeChangeValidator docTypeValidator; + + public Fixture(String currentSd, String nextSd) throws Exception { + super(currentSd, nextSd); + structFieldAttributeValidator = new StructFieldAttributeChangeValidator(currentDocType(), + currentDb().getDerivedConfiguration().getAttributeFields(), + nextDocType(), + nextDb().getDerivedConfiguration().getAttributeFields()); + docTypeValidator = new DocumentTypeChangeValidator(currentDocType(), nextDocType()); + } + + @Override + public List<VespaConfigChangeAction> validate() { + List<VespaConfigChangeAction> result = new ArrayList<>(); + result.addAll(structFieldAttributeValidator.validate(ValidationOverrides.empty, Instant.now())); + result.addAll(docTypeValidator.validate(ValidationOverrides.empty, Instant.now())); + return result; + } + } + + private static void validate(String currentSd, String nextSd) throws Exception { + new Fixture(currentSd, nextSd).assertValidation(); + } + + private static void validate(String currentSd, String nextSd, VespaConfigChangeAction expAction) throws Exception { + new Fixture(currentSd, nextSd).assertValidation(expAction); + } + + @Test + public void adding_attribute_aspect_to_struct_field_requires_refeed() throws Exception { + validate(arrayOfStruct(oneFieldStruct(), ""), + arrayOfStruct(oneFieldStruct(), structAttribute("s1")), + newRefeedAction("field-type-change", "Field 'f1.s1' changed: add attribute aspect")); + + validate(mapOfStruct(oneFieldStruct(), ""), + mapOfStruct(oneFieldStruct(), structAttribute("key")), + newRefeedAction("field-type-change", "Field 'f1.key' changed: add attribute aspect")); + + validate(mapOfStruct(oneFieldStruct(), ""), + mapOfStruct(oneFieldStruct(), structAttribute("value.s1")), + newRefeedAction("field-type-change", "Field 'f1.value.s1' changed: add attribute aspect")); + + validate(mapOfPrimitive(""), mapOfPrimitive(structAttribute("key")), + newRefeedAction("field-type-change", "Field 'f1.key' changed: add attribute aspect")); + + validate(mapOfPrimitive(""), mapOfPrimitive(structAttribute("value")), + newRefeedAction("field-type-change", "Field 'f1.value' changed: add attribute aspect")); + } + + @Test + public void removing_attribute_aspect_from_struct_field_is_ok() throws Exception { + validate(arrayOfStruct(oneFieldStruct(), structAttribute("s1")), + arrayOfStruct(oneFieldStruct(), "")); + + validate(mapOfStruct(oneFieldStruct(), structAttribute("key")), + mapOfStruct(oneFieldStruct(), "")); + + validate(mapOfStruct(oneFieldStruct(), structAttribute("value.s1")), + mapOfStruct(oneFieldStruct(), "")); + + validate(mapOfPrimitive(structAttribute("key")), mapOfPrimitive("")); + + validate(mapOfPrimitive(structAttribute("value")), mapOfPrimitive("")); + } + + @Test + public void adding_struct_field_with_attribute_aspect_is_ok() throws Exception { + validate(arrayOfStruct(oneFieldStruct(), ""), + arrayOfStruct(twoFieldStruct(), structAttribute("s2"))); + + validate(mapOfStruct(oneFieldStruct(), ""), + mapOfStruct(twoFieldStruct(), structAttribute("value.s2"))); + } + + @Test + public void removing_struct_field_with_attribute_aspect_is_ok() throws Exception { + validate(arrayOfStruct(twoFieldStruct(), structAttribute("s2")), + arrayOfStruct(oneFieldStruct(), "")); + + validate(mapOfStruct(twoFieldStruct(), structAttribute("value.s2")), + mapOfStruct(oneFieldStruct(), "")); + } + + @Test + public void adding_struct_field_without_attribute_aspect_is_ok() throws Exception { + validate(arrayOfStruct(oneFieldStruct(), ""), + arrayOfStruct(twoFieldStruct(), "")); + + validate(mapOfStruct(oneFieldStruct(), ""), + mapOfStruct(twoFieldStruct(), "")); + } + + @Test + public void removing_struct_field_without_attribute_aspect_is_ok() throws Exception { + validate(arrayOfStruct(twoFieldStruct(), ""), + arrayOfStruct(oneFieldStruct(), "")); + + validate(mapOfStruct(twoFieldStruct(), ""), + mapOfStruct(oneFieldStruct(), "")); + } + + private static String oneFieldStruct() { + return "struct s { field s1 type string {} }"; + } + + private static String twoFieldStruct() { + return "struct s { field s1 type string {} field s2 type int {} }"; + } + + private static String structAttribute(String fieldName) { + return "struct-field " + fieldName + " { indexing: attribute }"; + } + + private static String arrayOfStruct(String struct, String structField) { + return struct + " field f1 type array<s> { " + structField + "}"; + } + + private static String mapOfStruct(String struct, String structField) { + return struct + " field f1 type map<string,s> { " + structField + " }"; + } + + private static String mapOfPrimitive(String structField) { + return "field f1 type map<string,int> { " + structField + " }"; + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index c7e71c8c38d..c734addb8ce 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -37,6 +37,8 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { "/custom-handler/", "/search/", "/feed/", + "/remove/", + "/removelocation/", "/get/", "/visit/", "/document/", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index e318d4d6828..8158db51271 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -75,6 +75,8 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandler", "feed"); verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerCompatibility", "document"); verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerGet", "get"); + verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerRemove", "remove"); + verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", "removelocation"); verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerStatus", "feedstatus"); verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerVisit", "visit"); verifyCustomBindings("com.yahoo.vespa.http.server.FeedHandler", ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"); @@ -128,6 +130,20 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().contains("https://*/get/"), is(true)); assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().size(), equalTo(4)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("http://*/remove"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("https://*/remove"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("http://*/remove/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("https://*/remove/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().size(), equalTo(4)); + + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("http://*/removelocation"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("https://*/removelocation"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("http://*/removelocation/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("https://*/removelocation/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().size(), equalTo(4)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus"), not(nullValue())); assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().contains("http://*/feedstatus"), is(true)); assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().contains("https://*/feedstatus"), is(true)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 93e434c3b87..821162353d6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -198,6 +198,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * was not present for this id (meaning it either is not active or active on another * node in the config server cluster) */ + @Override public Optional<com.yahoo.config.provision.Deployment> deployFromLocalActive(ApplicationId application) { return deployFromLocalActive(application, Duration.ofSeconds(configserverConfig.zookeeper().barrierTimeout()).plus(Duration.ofSeconds(5))); } @@ -410,7 +411,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public void validateThatRemoteSessionIsPrepared(Tenant tenant, long sessionId) { RemoteSession session = getRemoteSession(tenant, sessionId); - if (!Session.Status.PREPARE.equals(session.getStatus())) + if ( ! Session.Status.PREPARE.equals(session.getStatus())) throw new IllegalStateException("Session not prepared: " + sessionId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java index 6828204b17c..e7f048c28e8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java @@ -39,5 +39,6 @@ public interface GlobalComponentRegistry { Optional<Provisioner> getHostProvisioner(); Zone getZone(); Clock getClock(); + ConfigServerDB getConfigServerDB(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java index 88f54e569df..e7862f29399 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java @@ -40,7 +40,9 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry private final HostRegistries hostRegistries; private final Optional<Provisioner> hostProvisioner; private final Zone zone; + private final ConfigServerDB configServerDB; + @SuppressWarnings("WeakerAccess") @Inject public InjectedGlobalComponentRegistry(Curator curator, ConfigCurator configCurator, @@ -54,7 +56,8 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry PermanentApplicationPackage permanentApplicationPackage, HostRegistries hostRegistries, HostProvisionerProvider hostProvisionerProvider, - Zone zone) { + Zone zone, + ConfigServerDB configServerDB) { this.curator = curator; this.configCurator = configCurator; this.metrics = metrics; @@ -68,6 +71,7 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry this.hostRegistries = hostRegistries; this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); this.zone = zone; + this.configServerDB = configServerDB; } @Override @@ -107,4 +111,7 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry @Override public Clock getClock() {return Clock.systemUTC();} + + @Override + public ConfigServerDB getConfigServerDB() { return configServerDB; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java index 59783bf1af5..773a4862033 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java @@ -12,10 +12,10 @@ import java.util.Collection; * reloaded. It only exists because the RpcServer cannot distinguish between a * successful reload of a new application and a reload of the same application. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public interface ReloadListener { + /** * Signal the listener that hosts used by by a particular tenant. * @@ -47,4 +47,5 @@ public interface ReloadListener { * Must be thread-safe. */ void applicationRemoved(ApplicationId applicationId); + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java index 293f35558cb..587015362c8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -19,8 +18,8 @@ public class TenantFileSystemDirs { private final File serverDB; private final TenantName tenant; - public TenantFileSystemDirs(ConfigserverConfig configserverConfig, TenantName tenant) { - this(new ConfigServerDB(configserverConfig).path(), tenant); + public TenantFileSystemDirs(ConfigServerDB configServerDB, TenantName tenant) { + this(configServerDB.path(), tenant); } // For testing diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index df2287c64cb..a8159405edd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -78,7 +78,7 @@ public class FileDirectory { return Files.walk(file.toPath(), 100).map(path -> { try { log.log(LogLevel.DEBUG, "Calculating hash for '" + path + "'"); - return hash(file, hasher); + return hash(path.toFile(), hasher); } catch (IOException e) { log.log(LogLevel.WARNING, "Failed getting hash from '" + path + "'"); return 0; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 42bf269e9d2..37379a9aadc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -87,8 +87,8 @@ public class FileServer { this.pullExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); } - public boolean hasFile(String fileName) { - return hasFile(new FileReference(fileName)); + public boolean hasFile(String fileReference) { + return hasFile(new FileReference(fileReference)); } private boolean hasFile(FileReference reference) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java index 131f5f6085b..4eefcc0ca75 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java @@ -1,14 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; - +import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.TenantName; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.http.HttpConfigResponse; import com.yahoo.vespa.config.server.http.SessionResponse; -import java.util.Collection; - /** * Tenant list response * @@ -16,17 +14,11 @@ import java.util.Collection; * */ public class ListTenantsResponse extends SessionResponse { - private final Collection<TenantName> tenantNames; - - public ListTenantsResponse(final Collection<TenantName> tenants) { + + ListTenantsResponse(ImmutableSet<TenantName> tenants) { super(); - this.tenantNames = tenants; Cursor tenantArray = this.root.setArray("tenants"); - synchronized (tenants) { - for (final TenantName tenantName : tenants) { - tenantArray.addString(tenantName.value()); - } - } + tenants.forEach(tenantName -> tenantArray.addString(tenantName.value())); } @Override @@ -34,7 +26,4 @@ public class ListTenantsResponse extends SessionResponse { return HttpConfigResponse.JSON_CONTENT_TYPE; } - public Collection<TenantName> getTenantNames() { - return tenantNames; - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java index c8e9da1265b..b79fd1c7a62 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.config.provision.TenantName; @@ -26,6 +27,7 @@ public class TenantHandler extends HttpHandler { private final TenantRepository tenantRepository; private final ApplicationRepository applicationRepository; + @SuppressWarnings("WeakerAccess") // instantiated by dependency injection @Inject public TenantHandler(Context ctx, TenantRepository tenantRepository, ApplicationRepository applicationRepository) { super(ctx); @@ -51,7 +53,7 @@ public class TenantHandler extends HttpHandler { Utils.checkThatTenantExists(tenantRepository, tenantName); return new TenantGetResponse(tenantName); } else if (isListTenantsRequest(request)) { - return new ListTenantsResponse(tenantRepository.getAllTenantNames()); + return new ListTenantsResponse(ImmutableSet.copyOf(tenantRepository.getAllTenantNames())); } else { throw new BadRequestException(request.getUri().toString()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 2a53f9ee45c..fd5d09a4ff2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -3,8 +3,6 @@ package com.yahoo.vespa.config.server.maintenance; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.session.FileDistributionFactory; import com.yahoo.vespa.curator.Curator; @@ -23,7 +21,7 @@ public class ConfigServerMaintenance extends AbstractComponent { Curator curator, FileDistributionFactory fileDistributionFactory) { DefaultTimes defaults = new DefaultTimes(configserverConfig); - tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval); + tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.defaultInterval); zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.defaultInterval); fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig); } @@ -42,18 +40,9 @@ public class ConfigServerMaintenance extends AbstractComponent { private static class DefaultTimes { private final Duration defaultInterval; - private final Duration tenantsMaintainerInterval; DefaultTimes(ConfigserverConfig configserverConfig) { - boolean isCd = configserverConfig.system().equals(SystemName.cd.name()); - boolean isTest = Environment.from(configserverConfig.environment()).isTest(); - this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes()); - // TODO: Want job control or feature flag to control when to run this, for now use a very - // long interval to avoid running the maintainer - this.tenantsMaintainerInterval = isCd || isTest || configserverConfig.region().equals("us-central-1") - ? defaultInterval - : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 454b98ddc05..9374d68a6ac 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.*; import com.yahoo.lang.SettableOptional; import com.yahoo.vespa.config.server.*; @@ -54,10 +55,15 @@ public class RemoteSession extends Session { } private ApplicationSet loadApplication() { + ApplicationPackage applicationPackage = zooKeeperClient.loadApplicationPackage(); + + // Read hosts allocated on the config server instance which created this + Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts(); + return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(), zooKeeperClient.readVespaVersion(), zooKeeperClient.loadApplicationPackage(), - new SettableOptional<>(), + new SettableOptional<>(allocatedHosts), clock.instant())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index ad967f49964..8c66b19653c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -160,7 +160,7 @@ public class TenantBuilder { private void createServerDbDirs() { if (tenantFileSystemDirs == null) { - tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigserverConfig(), tenant); + tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenant); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java index f4c0a9fd47a..8d7023fef5b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java @@ -85,6 +85,7 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host /** * Activates the config of the given app. Notifies listeners + * * @param applicationSet the {@link ApplicationSet} to be reloaded */ public void reloadConfig(ApplicationSet applicationSet) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java index 5ca3deab1fe..0c031556525 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java @@ -73,7 +73,8 @@ public class InjectedGlobalComponentRegistryTest { zone = Zone.defaultZone(); globalComponentRegistry = new InjectedGlobalComponentRegistry(curator, configCurator, metrics, modelFactoryRegistry, sessionPreparer, rpcServer, configserverConfig, - generationCounter, defRepo, permanentApplicationPackage, hostRegistries, hostProvisionerProvider, zone); + generationCounter, defRepo, permanentApplicationPackage, hostRegistries, hostProvisionerProvider, zone, + new ConfigServerDB(configserverConfig)); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index e4e45d3a014..d2965b8a489 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -51,6 +51,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private final Optional<Provisioner> hostProvisioner; private final Zone zone; private final Clock clock; + private final ConfigServerDB configServerDB; private TestComponentRegistry(Curator curator, ConfigCurator configCurator, Metrics metrics, ModelFactoryRegistry modelFactoryRegistry, @@ -82,6 +83,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { this.sessionPreparer = sessionPreparer; this.zone = zone; this.clock = clock; + this.configServerDB = new ConfigServerDB(configserverConfig); } public static class Builder { @@ -206,6 +208,9 @@ public class TestComponentRegistry implements GlobalComponentRegistry { } @Override public Clock getClock() { return clock;} + @Override + public ConfigServerDB getConfigServerDB() { return configServerDB;} + public FileDistributionFactory getFileDistributionFactory() { return fileDistributionFactory; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java index 88a6b1a0303..400ae4734af 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java @@ -18,6 +18,7 @@ import java.nio.file.attribute.FileTime; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; public class FileDirectoryTest { @@ -49,22 +50,29 @@ public class FileDirectoryTest { String subdirName = "subdir"; File subDirectory = new File(temporaryFolder.getRoot(), subdirName); - createFileInSubDir(subDirectory, "foo"); + createFileInSubDir(subDirectory, "foo", "some content"); FileReference fileReference = fileDirectory.addFile(subDirectory); File dir = fileDirectory.getFile(fileReference); assertTrue(dir.exists()); assertTrue(new File(dir, "foo").exists()); assertFalse(new File(dir, "doesnotexist").exists()); - assertEquals("1315a322fc323608", fileReference.value()); + assertEquals("bebc5a1aee74223d", fileReference.value()); + // Change contents of a file, file reference value should change + createFileInSubDir(subDirectory, "foo", "new content"); + FileReference fileReference2 = fileDirectory.addFile(subDirectory); + dir = fileDirectory.getFile(fileReference2); + assertTrue(new File(dir, "foo").exists()); + assertNotEquals(fileReference + " should not be equal to " + fileReference2, fileReference, fileReference2); + assertEquals("e5d4b3fe5ee3ede3", fileReference2.value()); // Add a file, should be available and file reference should have another value - createFileInSubDir(subDirectory, "bar"); - fileReference = fileDirectory.addFile(subDirectory); - dir = fileDirectory.getFile(fileReference); + createFileInSubDir(subDirectory, "bar", "some other content"); + FileReference fileReference3 = fileDirectory.addFile(subDirectory); + dir = fileDirectory.getFile(fileReference3); assertTrue(new File(dir, "foo").exists()); assertTrue(new File(dir, "bar").exists()); - assertEquals("9ca074b47a4b510c", fileReference.value()); + assertEquals("894bced3fc9d199b", fileReference3.value()); } @Test @@ -73,7 +81,7 @@ public class FileDirectoryTest { String subdirName = "subdir"; File subDirectory = new File(temporaryFolder.getRoot(), subdirName); - createFileInSubDir(subDirectory, "foo"); + createFileInSubDir(subDirectory, "foo", "some content"); FileReference fileReference = fileDirectory.addFile(subDirectory); File dir = fileDirectory.getFile(fileReference); assertTrue(dir.exists()); @@ -81,7 +89,7 @@ public class FileDirectoryTest { assertTrue(foo.exists()); FileTime fooCreatedTimestamp = Files.readAttributes(foo.toPath(), BasicFileAttributes.class).creationTime(); assertFalse(new File(dir, "doesnotexist").exists()); - assertEquals("1315a322fc323608", fileReference.value()); + assertEquals("bebc5a1aee74223d", fileReference.value()); // Remove a file, directory should be deleted before adding a new file try { Thread.sleep(1000);} catch (InterruptedException e) {/*ignore */} // Needed since we have timestamp resolution of 1 second @@ -95,7 +103,7 @@ public class FileDirectoryTest { // Check that creation timestamp is newer than the old one to be sure that a new file was written assertTrue(foo2CreatedTimestamp.compareTo(fooCreatedTimestamp) > 0); assertFalse(new File(dir, "doesnotexist").exists()); - assertEquals("1315a322fc323608", fileReference.value()); + assertEquals("bebc5a1aee74223d", fileReference.value()); } // Content in created file is equal to the filename string @@ -105,11 +113,11 @@ public class FileDirectoryTest { return fileDirectory.addFile(file); } - private void createFileInSubDir(File subDirectory, String filename) throws IOException { + private void createFileInSubDir(File subDirectory, String filename, String fileContent) throws IOException { if (!subDirectory.exists()) subDirectory.mkdirs(); File file = new File(subDirectory, filename); - IOUtils.writeFile(file, filename, false); + IOUtils.writeFile(file, fileContent, false); } } diff --git a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java index a9d6521e3eb..4cc3b48fd1a 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java @@ -1,6 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; +import com.google.common.util.concurrent.ForwardingExecutorService; +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.container.protect.ProcessTerminator; +import com.yahoo.jdisc.Metric; + import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; @@ -12,14 +20,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import com.google.inject.Inject; -import com.yahoo.container.protect.ProcessTerminator; -import com.google.common.util.concurrent.ForwardingExecutorService; -import com.yahoo.component.AbstractComponent; -import com.yahoo.concurrent.ThreadFactoryFactory; -import com.yahoo.container.di.componentgraph.Provider; -import com.yahoo.jdisc.Metric; - /** * A configurable thread pool provider. This provides the worker threads used for normal request processing. * Request an Executor injected in your component constructor if you want to use it. @@ -139,7 +139,7 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex super.execute(command); } catch (RejectedExecutionException e) { metric.add(MetricNames.REJECTED_REQUEST, 1, null); - long timeSinceLastReturnedThreadMillis = System.currentTimeMillis() - wrapped.lastThreadReturnTimeMillis; + long timeSinceLastReturnedThreadMillis = System.currentTimeMillis() - wrapped.lastThreadAssignmentTimeMillis; if (timeSinceLastReturnedThreadMillis > maxThreadExecutionTimeMillis) processTerminator.logAndDie("No worker threads have been available for " + timeSinceLastReturnedThreadMillis + " ms. Shutting down.", true); @@ -161,7 +161,7 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex /** A thread pool executor which maintains the last time a worker completed */ private final static class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor { - volatile long lastThreadReturnTimeMillis = System.currentTimeMillis(); + volatile long lastThreadAssignmentTimeMillis = System.currentTimeMillis(); private final AtomicLong startedCount = new AtomicLong(0); private final AtomicLong completedCount = new AtomicLong(0); @@ -177,13 +177,13 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex @Override protected void beforeExecute(Thread t, Runnable r) { super.beforeExecute(t, r); + lastThreadAssignmentTimeMillis = System.currentTimeMillis(); startedCount.incrementAndGet(); } @Override protected void afterExecute(Runnable r, Throwable t) { super.afterExecute(r, t); - lastThreadReturnTimeMillis = System.currentTimeMillis(); completedCount.incrementAndGet(); } diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 90836b1c07f..482c1c395de 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -459,7 +459,7 @@ <properties> <bouncycastle.version>1.58</bouncycastle.version> - <felix.version>5.4.0</felix.version> + <felix.version>5.6.10</felix.version> <findbugs.version>1.3.9</findbugs.version> <guava.version>18.0</guava.version> <guice.version>3.0</guice.version> diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceOrProviderClassVisitorTest.java b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceOrProviderClassVisitorTest.java new file mode 100644 index 00000000000..394f1b858b2 --- /dev/null +++ b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceOrProviderClassVisitorTest.java @@ -0,0 +1,78 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.servlet.jersey.classvisitor; + +import com.yahoo.container.servlet.jersey.ResourceOrProviderClassVisitor; +import org.junit.Test; +import org.objectweb.asm.ClassReader; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class ResourceOrProviderClassVisitorTest { + @Test + public void resource_is_detected() throws Exception { + assert_is_accepted(com.yahoo.container.servlet.jersey.classvisitor.Resource.class); + } + + @Test + public void provider_is_detected() throws Exception { + assert_is_accepted(com.yahoo.container.servlet.jersey.classvisitor.Provider.class); + } + + @Test + public void inner_class_is_ignored() throws Exception { + assert_is_ignored(com.yahoo.container.servlet.jersey.classvisitor.InnerClass.Inner.class); + } + + @Test + public void nested_public_class_is_detected() throws Exception { + assert_is_accepted(com.yahoo.container.servlet.jersey.classvisitor.NestedClass.Nested.class); + } + + @Test + public void nested_non_public_class_is_ignored() throws Exception { + assert_is_ignored(com.yahoo.container.servlet.jersey.classvisitor.NonPublicNestedClass.Nested.class); + } + + @Test + public void resource_with_multiple_annotations_is_detected() throws Exception { + assert_is_accepted(com.yahoo.container.servlet.jersey.classvisitor.ResourceWithMultipleAnnotations.class); + } + + @Test + public void interface_is_ignored() throws Exception { + assert_is_ignored(com.yahoo.container.servlet.jersey.classvisitor.InterfaceResource.class); + } + + @Test + public void abstract_class_is_ignored() throws Exception { + assert_is_ignored(com.yahoo.container.servlet.jersey.classvisitor.AbstractResource.class); + } + + @Test + public void className_is_equal_to_getName() throws Exception { + assertThat(analyzeClass(com.yahoo.container.servlet.jersey.classvisitor.Resource.class).getClassName(), is(com.yahoo.container.servlet.jersey.classvisitor.Resource.class.getName())); + } + + public void assert_is_accepted(Class<?> clazz) throws Exception { + assertTrue(className(clazz) + " was not accepted", + analyzeClass(clazz).isJerseyClass()); + } + + public void assert_is_ignored(Class<?> clazz) throws Exception { + assertFalse(className(clazz) + " was not ignored", + analyzeClass(clazz).isJerseyClass()); + } + + public ResourceOrProviderClassVisitor analyzeClass(Class<?> clazz) throws Exception { + return ResourceOrProviderClassVisitor.visit(new ClassReader(className(clazz))); + } + + public String className(Class<?> clazz) { + return clazz.getName(); + } +} + + diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceOrProviderClassVisitorTest.scala b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceOrProviderClassVisitorTest.scala deleted file mode 100644 index f20c5e02e62..00000000000 --- a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceOrProviderClassVisitorTest.scala +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.classvisitor - -import com.yahoo.container.servlet.jersey.{ResourceOrProviderClassVisitor, classvisitor} -import org.junit.{Assert, Test} -import org.objectweb.asm.ClassReader - -import Assert.assertThat -import org.hamcrest.CoreMatchers.is - -import scala.reflect.ClassTag - -class ResourceOrProviderClassVisitorTest { - @Test - def resource_is_detected() { - assert_is_accepted[classvisitor.Resource] - } - - @Test - def provider_is_detected() { - assert_is_accepted[classvisitor.Provider] - } - - @Test - def inner_class_is_ignored() { - assert_is_ignored[classvisitor.InnerClass#Inner] - } - - @Test - def nested_public_class_is_detected() { - assert_is_accepted[classvisitor.NestedClass.Nested] - } - - @Test - def nested_non_public_class_is_ignored() { - assert_is_ignored[classvisitor.NonPublicNestedClass.Nested] - } - - @Test - def resource_with_multiple_annotations_is_detected() { - assert_is_accepted[classvisitor.ResourceWithMultipleAnnotations] - } - - def interface_is_ignored() { - assert_is_ignored[classvisitor.InterfaceResource] - } - - @Test - def abstract_class_is_ignored() { - assert_is_ignored[classvisitor.AbstractResource] - } - - @Test - def className_is_equal_to_getName() { - assertThat(analyzeClass[classvisitor.Resource].getClassName, is(classOf[classvisitor.Resource].getName)) - } - - def assert_is_accepted[T: ClassTag] { - Assert.assertTrue(className[T] + " was not accepted", - analyzeClass[T].isJerseyClass) - } - - def assert_is_ignored[T: ClassTag] { - Assert.assertFalse(className[T] + " was not ignored", - analyzeClass[T].isJerseyClass) - } - - def analyzeClass[T: ClassTag] = { - ResourceOrProviderClassVisitor.visit(new ClassReader(className[T])) - } - - def className[T: ClassTag] = implicitly[ClassTag[T]].runtimeClass.getName -} - - diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 1160ea0a204..3a2e922f9e0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -50,9 +50,9 @@ public class FastHit extends Hit { * Summaries added to this hit which are not yet decoded into fields. * Fields are resolved by returning the first non-null value found by * 1) the field value from the Map of fields in the Hit supertype, and - * 2) each of the summaries, in the order of the list (which is the add order). + * 2) each of the summaries, reverse add order * This ensures that values set from code overwrites any value received as - * summary data. + * summary data, and fetching a new summary overrides previous summaries. * * The reason we keep this rather than eagerly decoding into a the field map * is to reduce garbage collection and decoding cost, with the assumption @@ -163,7 +163,7 @@ public class FastHit extends Hit { public void addSummary(DocsumDefinition docsumDef, Inspector value) { if (removedFields != null) removedFields.removeAll(docsumDef.fieldNames()); - summaries.add(new SummaryData(this, docsumDef, value, summaries.size())); + summaries.add(0, new SummaryData(this, docsumDef, value, 1 + summaries.size())); } /** @@ -331,6 +331,7 @@ public class FastHit extends Hit { private Object getSummaryValue(String name) { if (removedFields != null && removedFields.contains(name)) return null; + // fetch from last added summary with the field for (SummaryData summaryData : summaries) { Object value = summaryData.getField(name); if (value != null) return value; @@ -520,7 +521,7 @@ public class FastHit extends Hit { private final DocsumDefinition type; private final Inspector data; - /** The index of this summary in the list of summaries added to this */ + /** The index from the end of this summary in the list of summaries */ private final int index; SummaryData(FastHit hit, DocsumDefinition type, Inspector data, int index) { @@ -577,11 +578,11 @@ public class FastHit extends Hit { /** * Returns whether this field is present in the map properties - * or an earlier (lower index) summary in this hit + * or a summary added later in this hit */ private boolean shadowed(String name) { if (hit.hasField(name)) return true; - for (int i = 0; i < index; i++) { + for (int i = 0; i < hit.summaries.size() - index; i++) { if (hit.summaries.get(i).type.fieldNames().contains(name)) return true; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 708c48f0954..902fefae9be 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -35,6 +35,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { /** * Make a WAND item with no children. You can mention a common index or you can mention it on each child. + * * @param index The index it shall search. * @param N the target for minimum number of hits to produce; * a backend will not suppress any hits in the operator @@ -94,6 +95,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { * This threshold is currently only used if the WeakAndItem is searching a RISE index field. * The score threshold then specifies the minimum dot product score a match needs to be part of the result set. * Default value is 0. + * * @param scoreThreshold the score threshold. */ public void setScoreThreshold(int scoreThreshold) { diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 55c846ccb5b..7d60d7cf9ee 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -9,7 +9,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.yahoo.data.JsonProducer; import com.yahoo.data.access.Inspectable; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.Type; import com.yahoo.data.access.simple.JsonRender; +import com.yahoo.data.access.simple.Value; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; @@ -741,6 +744,34 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return true; } + private static Inspector wrapAsMap(Inspector data) { + if (data.type() != Type.ARRAY) return null; + if (data.entryCount() == 0) return null; + Value.ObjectValue map = new Value.ObjectValue(); + for (int i = 0; i < data.entryCount(); i++) { + Inspector obj = data.entry(i); + if (obj.type() != Type.OBJECT) return null; + if (obj.fieldCount() != 2) return null; + Inspector key = obj.field("key"); + Inspector value = obj.field("value"); + if (key.type() != Type.STRING) return null; + if (! value.valid()) return null; + map.put(key.asString(), value); + } + return map; + } + + private void renderInspector(Inspector data) throws IOException { + StringBuilder intermediate = new StringBuilder(); + Inspector asMap = wrapAsMap(data); + if (asMap != null) { + JsonRender.render(asMap, intermediate, true); + } else { + JsonRender.render(data, intermediate, true); + } + generator.writeRawValue(intermediate.toString()); + } + private void renderFieldContents(Object field) throws IOException { if (field == null) { generator.writeNull(); @@ -750,12 +781,10 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { generator.writeTree((TreeNode) field); } else if (field instanceof Tensor) { renderTensor(Optional.of((Tensor)field)); + } else if (field instanceof Inspectable) { + renderInspector(((Inspectable)field).inspect()); } else if (field instanceof JsonProducer) { generator.writeRawValue(((JsonProducer) field).toJson()); - } else if (field instanceof Inspectable) { - StringBuilder intermediate = new StringBuilder(); - JsonRender.render((Inspectable) field, intermediate, true); - generator.writeRawValue(intermediate.toString()); } else if (field instanceof StringFieldValue) { generator.writeString(((StringFieldValue)field).getString()); } else if (field instanceof TensorFieldValue) { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index 421f70c8d07..81f16f7f261 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -35,9 +35,11 @@ import static org.junit.Assert.fail; public class SlimeSummaryTestCase { - private static final String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; - private static final String partial_summary1_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg"; - private static final String partial_summary2_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg"; + private static final String cf_pre = "file:src/test/java/com/yahoo/prelude/fastsearch/"; + private static final String summary_cf = cf_pre + "summary.cfg"; + private static final String partial_summary1_cf = cf_pre + "partial-summary1.cfg"; + private static final String partial_summary2_cf = cf_pre + "partial-summary2.cfg"; + private static final String partial_summary3_cf = cf_pre + "partial-summary3.cfg"; @Test public void testDecodingEmpty() { @@ -143,6 +145,7 @@ public class SlimeSummaryTestCase { public void testFieldAccessAPI() { DocsumDefinitionSet partialDocsum1 = createDocsumDefinitionSet(partial_summary1_cf); DocsumDefinitionSet partialDocsum2 = createDocsumDefinitionSet(partial_summary2_cf); + DocsumDefinitionSet partialDocsum3 = createDocsumDefinitionSet(partial_summary3_cf); DocsumDefinitionSet fullDocsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); Map<String, Object> expected = new HashMap<>(); @@ -273,6 +276,18 @@ public class SlimeSummaryTestCase { expected.put("string_field", "string_value"); expected.put("longstring_field", "longstring_value"); assertFields(expected, hit); + + hit.removeField("string_field"); + hit.removeField("integer_field"); + partialDocsum3.lazyDecode("partial3", partialSummary3(), hit); + expected.put("string_field", "new str val"); + expected.put("integer_field", 5); + assertFields(expected, hit); + + hit.removeField("integer_field"); + partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); + expected.put("integer_field", 4); + assertFields(expected, hit); } @@ -344,6 +359,14 @@ public class SlimeSummaryTestCase { return encode((slime)); } + private byte[] partialSummary3() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + docsum.setString("string_field", "new str val"); + docsum.setLong("integer_field", 5); + return encode((slime)); + } + private byte[] fullishSummary() { Slime slime = new Slime(); Cursor docsum = slime.setObject(); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary3.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary3.cfg new file mode 100644 index 00000000000..5d7319fd393 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary3.cfg @@ -0,0 +1,10 @@ +documentdb[1] +documentdb[0].name test +documentdb[0].summaryclass[1] +documentdb[0].summaryclass[0].name partial3 +documentdb[0].summaryclass[0].id 3 +documentdb[0].summaryclass[0].fields[3] +documentdb[0].summaryclass[0].fields[0].name integer_field +documentdb[0].summaryclass[0].fields[0].type integer +documentdb[0].summaryclass[0].fields[1].name string_field +documentdb[0].summaryclass[0].fields[1].type string diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index bf56ad19f44..caad1c76362 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.component.chain.Chain; +import com.yahoo.data.access.simple.Value; import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.document.DataType; import com.yahoo.document.DocumentId; @@ -1111,6 +1112,53 @@ public class JsonRendererTestCase { } @Test + public final void testMapInField() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"structured\": {\n" + + " \"foo\": \"string foo\",\n" + + " \"bar\": [\"array bar elem 1\", \"array bar elem 2\"],\n" + + " \"baz\": {\"f1\": \"object baz field 1\", \"f2\": \"object baz field 2\"}\n" + + " }\n" + + " },\n" + + " \"id\": \"MapInField\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("MapInField"); + Value.ArrayValue atop = new Value.ArrayValue(); + atop.add(new Value.ObjectValue() + .put("key", new Value.StringValue("foo")) + .put("value", new Value.StringValue("string foo"))) + .add(new Value.ObjectValue() + .put("key", new Value.StringValue("bar")) + .put("value", new Value.ArrayValue() + .add(new Value.StringValue("array bar elem 1")) + .add(new Value.StringValue("array bar elem 2")))) + .add(new Value.ObjectValue() + .put("key", new Value.StringValue("baz")) + .put("value", new Value.ObjectValue() + .put("f1", new Value.StringValue("object baz field 1")) + .put("f2", new Value.StringValue("object baz field 2")))); + h.setField("structured", atop); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test public void testThatTheJsonValidatorCanCatchErrors() { String json = "{" + " \"root\": {" diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BrooklynStatusResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BrooklynStatusResource.java deleted file mode 100644 index c77f9fceef9..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BrooklynStatusResource.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.bcp; - -import com.fasterxml.jackson.databind.JsonNode; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; - -/** - * @author andreer - */ -@Path("") //Ensures that the produces annotation is inherited -@Produces(MediaType.APPLICATION_JSON) -public interface BrooklynStatusResource { - - @GET - @Path("{rotation}") - @Produces(MediaType.APPLICATION_JSON) - JsonNode rotationStatus(@PathParam("rotation") String page); -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java index bb2de87796b..e91a5909f80 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.config.provision.ApplicationId; /** - * @author jvenstad + * @author jonmv */ public interface BuildService { @@ -20,6 +20,11 @@ public interface BuildService { */ JobState stateOf(BuildJob buildJob); + /** + * Returns whether the given build job should be performed by this build service. + */ + default boolean builds(BuildJob buildJob) { return true; } + enum JobState { /** Job is not running, and may be triggered. */ @@ -37,6 +42,8 @@ public interface BuildService { } + // TODO jvenstad: Argh, refactor this, considering the new JobId, etc.. + // TODO jvenstad: Probably: make jobName JobType instead. class BuildJob { private final ApplicationId applicationId; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java new file mode 100644 index 00000000000..0a7bacefffb --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java @@ -0,0 +1,19 @@ +package com.yahoo.vespa.hosted.controller.api.integration; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; + +// TODO jvenstad: Change most of this. +public interface LogStore { + + /** Returns the log of the given deployment job. */ + String getTestLog(ApplicationId application, JobType jobType, int build); + + /** Stores the given log for the given deployment job. */ + void setTestLog(ApplicationId application, JobType jobType, int build, String log); + + /** Deletes the log for the given deployment job. */ + void deleteTestLog(ApplicationId application, JobType jobType, int build); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 741fee0210c..8147646bf06 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -19,19 +19,10 @@ public interface NodeRepository { /** List all nodes in zone owned by given application */ List<Node> list(ZoneId zone, ApplicationId application); - /** List all operational nodes in zone owned by given application */ - default List<Node> listOperational(ZoneId zone, ApplicationId application) { + /** List all nodes in states, in zone owned by given application */ + default List<Node> list(ZoneId zone, ApplicationId application, List<Node.State> states) { return list(zone, application).stream() - .filter(node -> { - switch (node.state()) { - case ready: - case active: - case inactive: - case reserved: - return true; - } - return false; - }) + .filter(node -> states.contains(node.state())) .collect(Collectors.toList()); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java new file mode 100644 index 00000000000..c9c73c3844a --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -0,0 +1,97 @@ +package com.yahoo.vespa.hosted.controller.api.integration.deployment; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; + +import java.util.Optional; +import java.util.stream.Stream; + +/** Job types that exist in the build system */ +public enum JobType { +// | enum name ------------| job name ------------------| Zone in main system ---------------------------------------| Zone in CD system ------------------------------------------- + component ("component" , null , null ), + systemTest ("system-test" , ZoneId.from("test" , "us-east-1") , ZoneId.from("test" , "cd-us-central-1") ), + stagingTest ("staging-test" , ZoneId.from("staging", "us-east-3") , ZoneId.from("staging", "cd-us-central-1") ), + productionCorpUsEast1 ("production-corp-us-east-1" , ZoneId.from("prod" , "corp-us-east-1") , null ), + productionUsEast3 ("production-us-east-3" , ZoneId.from("prod" , "us-east-3") , null ), + productionUsWest1 ("production-us-west-1" , ZoneId.from("prod" , "us-west-1") , null ), + productionUsCentral1 ("production-us-central-1" , ZoneId.from("prod" , "us-central-1") , null ), + productionApNortheast1 ("production-ap-northeast-1" , ZoneId.from("prod" , "ap-northeast-1") , null ), + productionApNortheast2 ("production-ap-northeast-2" , ZoneId.from("prod" , "ap-northeast-2") , null ), + productionApSoutheast1 ("production-ap-southeast-1" , ZoneId.from("prod" , "ap-southeast-1") , null ), + productionEuWest1 ("production-eu-west-1" , ZoneId.from("prod" , "eu-west-1") , null ), + productionAwsUsEast1a ("production-aws-us-east-1a" , ZoneId.from("prod" , "aws-us-east-1a") , null ), + productionCdAwsUsEast1a("production-cd-aws-us-east-1a", null , ZoneId.from("prod" , "cd-aws-us-east-1a")), + productionCdUsCentral1 ("production-cd-us-central-1" , null , ZoneId.from("prod" , "cd-us-central-1") ), + productionCdUsCentral2 ("production-cd-us-central-2" , null , ZoneId.from("prod" , "cd-us-central-2") ); + + private final String jobName; + private final ImmutableMap<SystemName, ZoneId> zones; + + JobType(String jobName, ZoneId mainZone, ZoneId cdZone) { + this.jobName = jobName; + ImmutableMap.Builder<SystemName, ZoneId> builder = ImmutableMap.builder(); + if (mainZone != null) builder.put(SystemName.main, mainZone); + if (cdZone != null) builder.put(SystemName.cd, cdZone); + this.zones = builder.build(); + } + + public String jobName() { return jobName; } + + /** Returns the zone for this job in the given system, or empty if this job does not have a zone */ + public Optional<ZoneId> zone(SystemName system) { + return Optional.ofNullable(zones.get(system)); + } + + /** Returns whether this is a production job */ + public boolean isProduction() { return environment() == Environment.prod; } + + /** Returns whether this is an automated test job */ + public boolean isTest() { return environment() != null && environment().isTest(); } + + /** Returns the environment of this job type, or null if it does not have an environment */ + public Environment environment() { + switch (this) { + case component: return null; + case systemTest: return Environment.test; + case stagingTest: return Environment.staging; + default: return Environment.prod; + } + } + + /** Returns the region of this job type, or null if it does not have a region */ + public Optional<RegionName> region(SystemName system) { + return zone(system).map(ZoneId::region); + } + + public static Optional<JobType> fromOptionalJobName(String jobName) { + return Stream.of(values()) + .filter(jobType -> jobType.jobName.equals(jobName)) + .findAny(); + } + + public static JobType fromJobName(String jobName) { + return fromOptionalJobName(jobName) + .orElseThrow(() -> new IllegalArgumentException("Unknown job name '" + jobName + "'")); + } + + /** Returns the job type for the given zone */ + public static Optional<JobType> from(SystemName system, ZoneId zone) { + return Stream.of(values()) + .filter(job -> job.zone(system).filter(zone::equals).isPresent()) + .findAny(); + } + + /** Returns the job job type for the given environment and region or null if none */ + public static Optional<JobType> from(SystemName system, Environment environment, RegionName region) { + switch (environment) { + case test: return Optional.of(systemTest); + case staging: return Optional.of(stagingTest); + } + return from(system, ZoneId.from(environment, region)); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHubMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHubMock.java index 9a398ef7cb5..9825887dcf5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHubMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHubMock.java @@ -8,7 +8,7 @@ import java.util.Map; import java.util.UUID; /** - * @author jvenstad + * @author jonmv */ public class GitHubMock implements GitHub { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentIssues.java index fdebcca6d83..6888e8ac06d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentIssues.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentIssues.java @@ -12,7 +12,7 @@ import java.util.Optional; /** * Represents the people responsible for keeping Vespa up and running in a given organization, etc.. * - * @author jvenstad + * @author jonmv */ public interface DeploymentIssues { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java index 086487b8be7..a9bc7868f7a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java @@ -14,7 +14,7 @@ import java.util.Optional; * Represents an issue which needs to reported, typically from the controller, to a responsible party, * the identity of which is determined by the propertyId and, possibly, assignee fields. * - * @author jvenstad + * @author jonmv */ public class Issue { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueId.java index 84b441ff4a8..ccb65582d3c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueId.java @@ -7,7 +7,7 @@ import java.util.Objects; * Used to identify issues stored in some issue tracking system. * The {@code value()} and {@code from()} methods should be inverses. * - * @author jvenstad + * @author jonmv */ public class IssueId { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Organization.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Organization.java index 776002f31cb..6dccaec3b7a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Organization.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Organization.java @@ -12,7 +12,7 @@ import java.util.Optional; * Represents the humans who use this software, and their organization. * Lets the software report issues to its caretakers, and provides other useful human resource lookups. * - * @author jvenstad + * @author jonmv */ public interface Organization { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/OwnershipIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/OwnershipIssues.java index 91b5eb89c38..ee17859c0fb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/OwnershipIssues.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/OwnershipIssues.java @@ -14,7 +14,7 @@ import java.util.Optional; * with reassurance that any misbehaving applications will swiftly be dealt with. * Ignored confirmation requests are periodically redirected to humans of higher rank, until they are acknowledged. * - * @author jvenstad + * @author jonmv */ public interface OwnershipIssues { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/User.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/User.java index 82a86de3824..a88a9432c89 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/User.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/User.java @@ -6,7 +6,7 @@ import java.util.Objects; /** * Represents a human computer user, typically by UNIX account name. * - * @author jvenstad + * @author jonmv */ public class User { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java index b169194fd40..c5efffd979a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java @@ -26,7 +26,7 @@ import java.util.logging.Logger; * A memory backed implementation of the Issues API which logs changes and does nothing else. * * @author bratseth - * @author jvenstad + * @author jonmv */ public class LoggingDeploymentIssues implements DeploymentIssues { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockBuildService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockBuildService.java index a8320f56fff..2a8b06888b0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockBuildService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockBuildService.java @@ -13,7 +13,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.BuildService.Job import static com.yahoo.vespa.hosted.controller.api.integration.BuildService.JobState.running; /** - * @author jvenstad + * @author jonmv */ public class MockBuildService extends AbstractComponent implements BuildService { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilter.java index 1cb5cd22b8d..3bcd7298a15 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilter.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilter.java @@ -7,7 +7,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.zone; * The methods here return instances of {@link ZoneList}, which extends ZoneFilter, but with accessors and additional filters. * This forces the developer to consider which of the filters in this class to apply, prior to processing any zones. * - * @author jvenstad + * @author jonmv */ public interface ZoneFilter { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java index 7f4637e0492..67d2fd14e6b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java @@ -15,7 +15,7 @@ import java.util.stream.Collectors; /** * A Zones.List implementation which assumes all zones are controllerManaged. * - * @author jvenstad + * @author jonmv */ public class ZoneFilterMock implements ZoneList { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java index 21ac7a654b8..b53b81398c6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java @@ -11,7 +11,7 @@ import java.util.Objects; * * Serialised form is 'environment.region'. * - * @author jvenstad + * @author jonmv */ public class ZoneId { // TODO: Replace usages of environment + region with usages of this. diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java index 27e8a598043..4205f30e995 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java @@ -12,7 +12,7 @@ import java.util.List; * This is typically offered after an initial filter from {@link ZoneFilter} has been applied. * This forces the developer to consider which zones to process. * - * @author jvenstad + * @author jonmv */ public interface ZoneList extends ZoneFilter { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/nonpublic/HeaderFields.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/nonpublic/HeaderFields.java deleted file mode 100644 index 78a6750aedb..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/nonpublic/HeaderFields.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.nonpublic; - -/** - * Non public header fields that are not part of the public api. - * - * Placed here since this is the only module we own that both the - * command-line client and controller-server depend on. - * - * @author Tony Vaagenes - */ -public class HeaderFields { - public static final String USER_ID_HEADER_FIELD = "vespa.hosted.trusted.username"; -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/ContextAttributes.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/ContextAttributes.java deleted file mode 100644 index 1cdff0f920b..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/ContextAttributes.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.common; - -/** - * Constants for request context attributes used in our APIs. - * - * @author mpolden - */ -public interface ContextAttributes { - - String SECURITY_CONTEXT_ATTRIBUTE = "vespa.hosted.security_context"; - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/NotFoundCheckedException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/NotFoundCheckedException.java deleted file mode 100644 index a55a7e2bdfc..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/NotFoundCheckedException.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.common; - -/** - * We have tons of places where we throw exceptions when - * some hosted resource is not found. This is usually - * done with IllegalArgumentExceptions, java.ws.rs exceptions or - * the servermodel runtime exceptions in the controller-server module. - * - * This is a checked alternative to do the same thing. - * - * @author smorgrav - */ -public class NotFoundCheckedException extends Exception { - - public NotFoundCheckedException() { - super(); - } - - public NotFoundCheckedException(String msg) { - super(msg); - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/package-info.java deleted file mode 100644 index 95decd86e8b..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.common; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 295e0102782..9208537dd98 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; import java.util.Collections; import java.util.Comparator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index f0e278c3e6d..57708bfc89c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -37,7 +37,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; import com.yahoo.vespa.hosted.controller.application.SystemApplication; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index b65d3bc0849..3f5f3369c0a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -25,6 +25,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingSe import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.deployment.DelegatingBuildService; +import com.yahoo.vespa.hosted.controller.deployment.InternalBuildService; +import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.rotation.Rotation; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; @@ -61,6 +64,7 @@ public class Controller extends AbstractComponent { private final CuratorDb curator; private final ApplicationController applicationController; private final TenantController tenantController; + private final JobController jobController; private final Clock clock; private final GitHub gitHub; private final EntityService entityService; @@ -114,13 +118,15 @@ public class Controller extends AbstractComponent { this.clock = Objects.requireNonNull(clock, "Clock cannot be null"); this.athenzClientFactory = Objects.requireNonNull(athenzClientFactory, "AthenzClientFactory cannot be null"); + jobController = new JobController(this); applicationController = new ApplicationController(this, curator, athenzClientFactory, Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null"), Objects.requireNonNull(nameService, "NameService cannot be null"), configServer, Objects.requireNonNull(artifactRepository, "ArtifactRepository cannot be null"), Objects.requireNonNull(routingGenerator, "RoutingGenerator cannot be null"), - Objects.requireNonNull(buildService, "BuildService cannot be null"), + new DelegatingBuildService(Objects.requireNonNull(buildService, "BuildService cannot be null"), + new InternalBuildService(jobController)), clock); tenantController = new TenantController(this, curator, athenzClientFactory); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 3207d4b8399..79e7fa0295a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -18,7 +18,7 @@ import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -34,7 +34,7 @@ import java.util.OptionalLong; * An application that has been locked for modification. Provides methods for modifying an application's fields. * * @author mpolden - * @author jvenstad + * @author jonmv */ public class LockedApplication { @@ -158,7 +158,7 @@ public class LockedApplication { return with(deployments); } - public LockedApplication withoutDeploymentJob(DeploymentJobs.JobType jobType) { + public LockedApplication withoutDeploymentJob(JobType jobType) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs.without(jobType), change, outstandingChange, ownershipIssueId, metrics, rotation); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java index 271942ff9a3..a890c50a0af 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareRes * @author Oyvind Gronnesby */ public class ActivateResult { + // TODO jvenstad: Replace this class with just the PrepareResponse. private final RevisionId revisionId; private final PrepareResponse prepareResponse; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index d3193fd486d..4968e161a35 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -3,12 +3,9 @@ package com.yahoo.vespa.hosted.controller.application; import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import java.util.Collection; import java.util.HashMap; @@ -17,7 +14,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; -import java.util.stream.Stream; /** * Information about which deployment jobs an application should run and their current status. @@ -122,93 +118,6 @@ public class DeploymentJobs { return id; } - /** Job types that exist in the build system */ - public enum JobType { -// | enum name ------------| job name ------------------| Zone in main system ---------------------------------------| Zone in CD system ------------------------------------------- - component ("component" , null , null ), - systemTest ("system-test" , ZoneId.from("test" , "us-east-1") , ZoneId.from("test" , "cd-us-central-1")), - stagingTest ("staging-test" , ZoneId.from("staging", "us-east-3") , ZoneId.from("staging", "cd-us-central-1")), - productionCorpUsEast1 ("production-corp-us-east-1" , ZoneId.from("prod" , "corp-us-east-1") , null ), - productionUsEast3 ("production-us-east-3" , ZoneId.from("prod" , "us-east-3") , null ), - productionUsWest1 ("production-us-west-1" , ZoneId.from("prod" , "us-west-1") , null ), - productionUsCentral1 ("production-us-central-1" , ZoneId.from("prod" , "us-central-1") , null ), - productionApNortheast1 ("production-ap-northeast-1" , ZoneId.from("prod" , "ap-northeast-1") , null ), - productionApNortheast2 ("production-ap-northeast-2" , ZoneId.from("prod" , "ap-northeast-2") , null ), - productionApSoutheast1 ("production-ap-southeast-1" , ZoneId.from("prod" , "ap-southeast-1") , null ), - productionEuWest1 ("production-eu-west-1" , ZoneId.from("prod" , "eu-west-1") , null ), - productionAwsUsEast1a ("production-aws-us-east-1a" , ZoneId.from("prod" , "aws-us-east-1a") , null ), - productionCdAwsUsEast1a("production-cd-aws-us-east-1a" , null , ZoneId.from("prod" , "cd-aws-us-east-1a")), - productionCdUsCentral1 ("production-cd-us-central-1", null , ZoneId.from("prod" , "cd-us-central-1")), - productionCdUsCentral2 ("production-cd-us-central-2", null , ZoneId.from("prod" , "cd-us-central-2")); - - private final String jobName; - private final ImmutableMap<SystemName, ZoneId> zones; - - JobType(String jobName, ZoneId mainZone, ZoneId cdZone) { - this.jobName = jobName; - ImmutableMap.Builder<SystemName, ZoneId> builder = ImmutableMap.builder(); - if (mainZone != null) builder.put(SystemName.main, mainZone); - if (cdZone != null) builder.put(SystemName.cd, cdZone); - this.zones = builder.build(); - } - - public String jobName() { return jobName; } - - /** Returns the zone for this job in the given system, or empty if this job does not have a zone */ - public Optional<ZoneId> zone(SystemName system) { - return Optional.ofNullable(zones.get(system)); - } - - /** Returns whether this is a production job */ - public boolean isProduction() { return environment() == Environment.prod; } - - /** Returns whether this is an automated test job */ - public boolean isTest() { return environment() != null && environment().isTest(); } - - /** Returns the environment of this job type, or null if it does not have an environment */ - public Environment environment() { - switch (this) { - case component: return null; - case systemTest: return Environment.test; - case stagingTest: return Environment.staging; - default: return Environment.prod; - } - } - - /** Returns the region of this job type, or null if it does not have a region */ - public Optional<RegionName> region(SystemName system) { - return zone(system).map(ZoneId::region); - } - - public static Optional<JobType> fromOptionalJobName(String jobName) { - return Stream.of(values()) - .filter(jobType -> jobType.jobName.equals(jobName)) - .findAny(); - } - - public static JobType fromJobName(String jobName) { - return fromOptionalJobName(jobName) - .orElseThrow(() -> new IllegalArgumentException("Unknown job name '" + jobName + "'")); - } - - /** Returns the job type for the given zone */ - public static Optional<JobType> from(SystemName system, ZoneId zone) { - return Stream.of(values()) - .filter(job -> job.zone(system).filter(zone::equals).isPresent()) - .findAny(); - } - - /** Returns the job job type for the given environment and region or null if none */ - public static Optional<JobType> from(SystemName system, Environment environment, RegionName region) { - switch (environment) { - case test: return Optional.of(systemTest); - case staging: return Optional.of(stagingTest); - } - return from(system, ZoneId.from(environment, region)); - } - - } - /** A job report. This class is immutable. */ public static class JobReport { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java index 90435e13a1b..cd15556ba9b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; import java.time.Instant; @@ -16,7 +16,7 @@ import java.util.function.Predicate; /** * A list of deployment jobs that can be filtered in various ways. * - * @author jvenstad + * @author jonmv */ public class JobList { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java index f44d2218145..a06a3e00340 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java @@ -2,8 +2,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import java.time.Instant; import java.util.Objects; @@ -20,7 +19,7 @@ import static java.util.Objects.requireNonNull; */ public class JobStatus { - private final DeploymentJobs.JobType type; + private final JobType type; private final Optional<JobRun> lastTriggered; private final Optional<JobRun> lastCompleted; @@ -33,7 +32,7 @@ public class JobStatus { * Used by the persistence layer (only) to create a complete JobStatus instance. * Other creation should be by using initial- and with- methods. */ - public JobStatus(DeploymentJobs.JobType type, Optional<DeploymentJobs.JobError> jobError, + public JobStatus(JobType type, Optional<DeploymentJobs.JobError> jobError, Optional<JobRun> lastTriggered, Optional<JobRun> lastCompleted, Optional<JobRun> firstFailing, Optional<JobRun> lastSuccess) { requireNonNull(type, "jobType cannot be null"); @@ -47,14 +46,14 @@ public class JobStatus { this.jobError = jobError; // Never say we triggered component because we don't: - this.lastTriggered = type == DeploymentJobs.JobType.component ? Optional.empty() : lastTriggered; + this.lastTriggered = type == JobType.component ? Optional.empty() : lastTriggered; this.lastCompleted = lastCompleted; this.firstFailing = firstFailing; this.lastSuccess = lastSuccess; } /** Returns an empty job status */ - public static JobStatus initial(DeploymentJobs.JobType type) { + public static JobStatus initial(JobType type) { return new JobStatus(type, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } @@ -84,7 +83,7 @@ public class JobStatus { return new JobStatus(type, jobError, lastTriggered, Optional.of(completion), firstFailing, lastSuccess); } - public DeploymentJobs.JobType type() { return type; } + public JobType type() { return type; } /** Returns true unless this job last completed with a failure */ public boolean isSuccess() { @@ -94,6 +93,11 @@ public class JobStatus { /** The error of the last completion, or empty if the last run succeeded */ public Optional<DeploymentJobs.JobError> jobError() { return jobError; } + /** Returns whether this last failed on out of capacity */ + public boolean isOutOfCapacity() { + return jobError.filter(error -> error == DeploymentJobs.JobError.outOfCapacity).isPresent(); + } + /** * Returns the last triggering of this job, or empty if the controller has never triggered it * and not seen a deployment for it diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 1e96f33c275..98ec05e563a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import java.util.Arrays; import java.util.List; @@ -53,6 +54,11 @@ public enum SystemApplication { return Arrays.asList(values()); } + /** Node states to consider when upgrading */ + public static List<Node.State> activeStates() { + return Arrays.asList(Node.State.active, Node.State.reserved); + } + @Override public String toString() { return String.format("system application %s of type %s", id, nodeType); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java new file mode 100644 index 00000000000..d2159841c9d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java @@ -0,0 +1,30 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.vespa.hosted.controller.api.integration.BuildService; + +/** + * Sends build jobs to an internal build system whenever it accepts them, or to an external one otherwise. + * + * @author jonmv + */ +public class DelegatingBuildService implements BuildService { + + private final BuildService external; + private final BuildService internal; + + public DelegatingBuildService(BuildService external, BuildService internal) { + this.external = external; + this.internal = internal; + } + + @Override + public void trigger(BuildJob buildJob) { + (internal.builds(buildJob) ? internal : external).trigger(buildJob); + } + + @Override + public JobState stateOf(BuildJob buildJob) { + return (internal.builds(buildJob) ? internal : external).stateOf(buildJob); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java deleted file mode 100644 index 1c535a5a331..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; -import com.yahoo.vespa.hosted.controller.application.JobStatus; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.function.Supplier; - -import static java.util.Comparator.comparingInt; -import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.toList; - -/** - * This class determines the order of deployments according to an application's deployment spec. - * - * @author mpolden - */ -public class DeploymentOrder { - - private final Supplier<SystemName> system; - - public DeploymentOrder(Supplier<SystemName> system) { - this.system = Objects.requireNonNull(system, "system may not be null"); - } - - /** Returns jobs for given deployment spec, in the order they are declared */ - public List<JobType> jobsFrom(DeploymentSpec deploymentSpec) { - return deploymentSpec.steps().stream() - .flatMap(step -> step.zones().stream()) - .map(this::toJob) - .collect(collectingAndThen(toList(), Collections::unmodifiableList)); - } - - /** Returns job status sorted according to deployment spec */ - public List<JobStatus> sortBy(DeploymentSpec deploymentSpec, Collection<JobStatus> jobStatus) { - List<DeploymentJobs.JobType> sortedJobs = jobsFrom(deploymentSpec); - return jobStatus.stream() - .sorted(comparingInt(job -> sortedJobs.indexOf(job.type()))) - .collect(collectingAndThen(toList(), Collections::unmodifiableList)); - } - - /** Returns deployments sorted according to declared zones */ - public List<Deployment> sortBy(List<DeploymentSpec.DeclaredZone> zones, Collection<Deployment> deployments) { - List<ZoneId> productionZones = zones.stream() - .filter(z -> z.region().isPresent()) - .map(z -> ZoneId.from(z.environment(), z.region().get())) - .collect(toList()); - return deployments.stream() - .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone()))) - .collect(collectingAndThen(toList(), Collections::unmodifiableList)); - } - - /** Resolve job from deployment step */ - public JobType toJob(DeploymentSpec.DeclaredZone zone) { - return JobType.from(system.get(), zone.environment(), zone.region().orElse(null)) - .orElseThrow(() -> new IllegalArgumentException("Invalid zone " + zone)); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java new file mode 100644 index 00000000000..51c1e61ecb2 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java @@ -0,0 +1,113 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.JobStatus; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static java.util.Collections.singletonList; +import static java.util.Comparator.comparingInt; +import static java.util.stream.Collectors.collectingAndThen; + +/** + * This class provides helper methods for reading a deployment spec. + * + * @author mpolden + */ +public class DeploymentSteps { + + private final DeploymentSpec spec; + private final Supplier<SystemName> system; + + public DeploymentSteps(DeploymentSpec spec, Supplier<SystemName> system) { + this.spec = Objects.requireNonNull(spec, "spec cannot be null"); + this.system = Objects.requireNonNull(system, "system cannot be null"); + } + + /** Returns jobs for this, in the order they are declared */ + public List<JobType> jobs() { + return spec.steps().stream() + .flatMap(step -> step.zones().stream()) + .map(this::toJob) + .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + /** Returns job status sorted according to deployment spec */ + public List<JobStatus> sortBy(Collection<JobStatus> jobStatus) { + List<JobType> sortedJobs = jobs(); + return jobStatus.stream() + .sorted(comparingInt(job -> sortedJobs.indexOf(job.type()))) + .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + /** Returns deployments sorted according to declared zones */ + public List<Deployment> sortBy2(Collection<Deployment> deployments) { + List<ZoneId> productionZones = spec.zones().stream() + .filter(z -> z.region().isPresent()) + .map(z -> ZoneId.from(z.environment(), z.region().get())) + .collect(Collectors.toList()); + return deployments.stream() + .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone()))) + .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + /** Resolve jobs from step */ + public List<JobType> toJobs(DeploymentSpec.Step step) { + return step.zones().stream() + .map(this::toJob) + .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + /** Returns test jobs in this */ + public List<JobType> testJobs() { + return toJobs(test()); + } + + /** Returns production jobs in this */ + public List<JobType> productionJobs() { + return toJobs(production()); + } + + /** Returns test steps in this */ + public List<DeploymentSpec.Step> test() { + if (spec.steps().isEmpty()) { + return singletonList(new DeploymentSpec.DeclaredZone(Environment.test)); + } + return spec.steps().stream() + .filter(step -> step.deploysTo(Environment.test) || step.deploysTo(Environment.staging)) + .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + /** Returns production steps in this */ + public List<DeploymentSpec.Step> production() { + return spec.steps().stream() + .filter(step -> step.deploysTo(Environment.prod) || step.zones().isEmpty()) + .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + /** Resolve job from deployment zone */ + private JobType toJob(DeploymentSpec.DeclaredZone zone) { + return JobType.from(system.get(), zone.environment(), zone.region().orElse(null)) + .orElseThrow(() -> new IllegalArgumentException("Invalid zone " + zone)); + } + + /** Resolve jobs from steps */ + private List<JobType> toJobs(List<DeploymentSpec.Step> steps) { + return steps.stream() + .flatMap(step -> toJobs(step).stream()) + .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 63a6ac234ff..e3b4b4cef8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; -import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.DeploymentSpec.Step; import com.yahoo.config.provision.ApplicationId; @@ -15,9 +14,8 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; @@ -26,6 +24,7 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -33,21 +32,17 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; -import java.util.Set; import java.util.function.Supplier; import java.util.logging.Logger; import java.util.stream.Stream; -import static com.yahoo.config.provision.Environment.prod; -import static com.yahoo.config.provision.Environment.staging; -import static com.yahoo.config.provision.Environment.test; import static com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; import static com.yahoo.vespa.hosted.controller.api.integration.BuildService.JobState.idle; import static com.yahoo.vespa.hosted.controller.api.integration.BuildService.JobState.queued; import static com.yahoo.vespa.hosted.controller.api.integration.BuildService.JobState.running; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static java.util.Comparator.comparing; @@ -56,7 +51,6 @@ import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.partitioningBy; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; /** * Responsible for scheduling deployment jobs in a build system and keeping @@ -66,7 +60,7 @@ import static java.util.stream.Collectors.toSet; * * @author bratseth * @author mpolden - * @author jvenstad + * @author jonmv */ public class DeploymentTrigger { @@ -74,18 +68,16 @@ public class DeploymentTrigger { private final Controller controller; private final Clock clock; - private final DeploymentOrder order; private final BuildService buildService; public DeploymentTrigger(Controller controller, BuildService buildService, Clock clock) { this.controller = Objects.requireNonNull(controller, "controller cannot be null"); this.buildService = Objects.requireNonNull(buildService, "buildService cannot be null"); this.clock = Objects.requireNonNull(clock, "clock cannot be null"); - this.order = new DeploymentOrder(controller::system); } - public DeploymentOrder deploymentOrder() { - return order; + public DeploymentSteps steps(DeploymentSpec spec) { + return new DeploymentSteps(spec, controller::system); } /** @@ -109,8 +101,8 @@ public class DeploymentTrigger { JobRun triggering; if (report.jobType() == component) { ApplicationVersion applicationVersion = ApplicationVersion.from(report.sourceRevision().get(), report.buildNumber()); - triggering = JobRun.triggering(controller.systemVersion(), applicationVersion, Optional - .empty(), Optional.empty(), "Application commit", clock.instant()); + triggering = JobRun.triggering(controller.systemVersion(), applicationVersion, Optional.empty(), + Optional.empty(), "Application commit", clock.instant()); if (report.success()) { if (acceptNewApplicationVersion(application.get())) application = application.withChange(application.get().change().with(applicationVersion)) @@ -195,7 +187,8 @@ public class DeploymentTrigger { buildService.trigger(BuildJob.of(applicationId, application.deploymentJobs().projectId().getAsLong(), jobType.jobName())); return singletonList(component); } - Versions versions = versions(application, application.change(), deploymentFor(application, jobType)); + Versions versions = Versions.from(application.change(), application, deploymentFor(application, jobType), + controller.systemVersion()); String reason = "Job triggered manually by " + user; return (jobType.isProduction() && ! isTested(application, versions) ? testJobs(application, versions, reason, clock.instant()).stream() @@ -241,7 +234,7 @@ public class DeploymentTrigger { private Optional<JobRun> successOn(Application application, JobType jobType, Versions versions) { return application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::lastSuccess) - .filter(run -> targetsMatch(versions, run)); + .filter(versions::targetsMatch); } private Optional<Deployment> deploymentFor(Application application, JobType jobType) { @@ -279,47 +272,90 @@ public class DeploymentTrigger { .<Instant>flatMap(job -> job.lastSuccess().map(JobRun::at))); String reason = "New change available"; List<Job> testJobs = null; // null means "uninitialised", while empty means "don't run any jobs". + DeploymentSteps steps = steps(application.deploymentSpec()); - if (change.isPresent()) - for (Step step : productionStepsOf(application)) { - Set<JobType> stepJobs = step.zones().stream().map(order::toJob).collect(toSet()); - List<JobType> remainingJobs = stepJobs.stream().filter(job -> ! isComplete(change, application, job)).collect(toList()); - if ( ! remainingJobs.isEmpty()) { // Step is incomplete; trigger remaining jobs if ready, or their test jobs if untested. + if (change.isPresent()) { + for (Step step : steps.production()) { + List<JobType> stepJobs = steps.toJobs(step); + List<JobType> remainingJobs = stepJobs.stream().filter(job -> !isComplete(change, application, job)).collect(toList()); + if (!remainingJobs.isEmpty()) { // Step is incomplete; trigger remaining jobs if ready, or their test jobs if untested. for (JobType job : remainingJobs) { - Versions versions = versions(application, change, deploymentFor(application, job)); + Versions versions = Versions.from(change, application, deploymentFor(application, job), + controller.systemVersion()); if (isTested(application, versions)) { - if ( completedAt.isPresent() - && jobStateOf(application, job) == idle - && stepJobs.containsAll(runningProductionJobs(application))) + if (completedAt.isPresent() && canTrigger(job, versions, application, stepJobs)) { jobs.add(deploymentJob(application, versions, change, job, reason, completedAt.get())); - if ( ! alreadyTriggered(application, versions)) + } + if (!alreadyTriggered(application, versions)) { testJobs = emptyList(); - } - else if (testJobs == null) { - testJobs = testJobs(application, versions, String.format("Testing deployment for %s (%s)", job.jobName(), versions.toString()), - completedAt.orElse(clock.instant())); + } + } else if (testJobs == null) { + testJobs = testJobs(application, versions, + String.format("Testing deployment for %s (%s)", + job.jobName(), versions.toString()), + completedAt.orElseGet(clock::instant)); } } completedAt = Optional.empty(); - } - else { // All jobs are complete; find the time of completion of this step. + } else { // All jobs are complete; find the time of completion of this step. if (stepJobs.isEmpty()) { // No jobs means this is delay step. Duration delay = ((DeploymentSpec.Delay) step).duration(); - completedAt = completedAt.map(at -> at.plus(delay)).filter(at -> ! at.isAfter(clock.instant())); + completedAt = completedAt.map(at -> at.plus(delay)).filter(at -> !at.isAfter(clock.instant())); reason += " after a delay of " + delay; - } - else { + } else { completedAt = stepJobs.stream().map(job -> application.deploymentJobs().statusOf(job).get().lastCompleted().get().at()).max(naturalOrder()); reason = "Available change in " + stepJobs.stream().map(JobType::jobName).collect(joining(", ")); } } } - if (testJobs == null) - testJobs = testJobs(application, versions(application, application.change(), Optional.empty()), + } + if (testJobs == null) { + testJobs = testJobs(application, Versions.from(application, controller.systemVersion()), "Testing last changes outside prod", clock.instant()); + } jobs.addAll(testJobs); }); - return jobs; + return Collections.unmodifiableList(jobs); + } + + /** Returns whether given job should be triggered */ + private boolean canTrigger(JobType job, Versions versions, Application application, List<JobType> parallelJobs) { + if (jobStateOf(application, job) != idle) return false; + if (parallelJobs != null && !parallelJobs.containsAll(runningProductionJobs(application))) return false; + + return triggerAt(clock.instant(), job, versions, application); + } + + /** Returns whether given job should be triggered */ + private boolean canTrigger(JobType job, Versions versions, Application application) { + return canTrigger(job, versions, application, null); + } + + /** Returns whether job can trigger at given instant */ + private boolean triggerAt(Instant instant, JobType job, Versions versions, Application application) { + Optional<JobStatus> jobStatus = application.deploymentJobs().statusOf(job); + if (!jobStatus.isPresent()) return true; + if (jobStatus.get().isSuccess()) return true; // Success + if (!jobStatus.get().lastCompleted().isPresent()) return true; // Never completed + if (!jobStatus.get().firstFailing().isPresent()) return true; // Should not happen as firstFailing should be set for an unsuccessful job + if (!versions.targetsMatch(jobStatus.get().lastCompleted().get())) return true; // Always trigger as targets have changed + + Instant firstFailing = jobStatus.get().firstFailing().get().at(); + Instant lastCompleted = jobStatus.get().lastCompleted().get().at(); + + // Retry all errors immediately for 1 minute + if (firstFailing.isAfter(instant.minus(Duration.ofMinutes(1)))) return true; + + // Retry out of capacity errors in test environments every minute + if (job.isTest() && jobStatus.get().isOutOfCapacity()) { + return lastCompleted.isBefore(instant.minus(Duration.ofMinutes(1))); + } + + // Retry other errors + if (firstFailing.isAfter(instant.minus(Duration.ofHours(1)))) { // If we failed within the last hour ... + return lastCompleted.isBefore(instant.minus(Duration.ofMinutes(10))); // ... retry every 10 minutes + } + return lastCompleted.isBefore(instant.minus(Duration.ofHours(2))); // Retry at most every 2 hours } // ---------- Job state helpers ---------- @@ -358,7 +394,7 @@ public class DeploymentTrigger { */ private boolean isComplete(Change change, Application application, JobType jobType) { Optional<Deployment> existingDeployment = deploymentFor(application, jobType); - return successOn(application, jobType, versions(application, change, existingDeployment)).isPresent() + return successOn(application, jobType, Versions.from(change, application, existingDeployment, controller.systemVersion())).isPresent() || jobType.isProduction() && existingDeployment.map(deployment -> ! isUpgrade(change, deployment) && isDowngrade(application.change(), deployment)) .orElse(false); @@ -379,7 +415,7 @@ public class DeploymentTrigger { private Optional<Instant> testedAt(Application application, Versions versions) { Optional<JobRun> testRun = successOn(application, systemTest, versions); Optional<JobRun> stagingRun = successOn(application, stagingTest, versions) - .filter(run -> sourcesMatchIfPresent(versions, run)); + .filter(versions::sourcesMatchIfPresent); return max(testRun.map(JobRun::at), stagingRun.map(JobRun::at)) .filter(__ -> testRun.isPresent() && stagingRun.isPresent()); } @@ -388,24 +424,11 @@ public class DeploymentTrigger { return application.deploymentJobs().jobStatus().values().stream() .filter(job -> job.type().isProduction()) .anyMatch(job -> job.lastTriggered() - .filter(run -> targetsMatch(versions, run)) - .filter(run -> sourcesMatchIfPresent(versions, run)) + .filter(versions::targetsMatch) + .filter(versions::sourcesMatchIfPresent) .isPresent()); } - /** If the given state's sources are present and differ from its targets, returns whether they are equal to those - * of the given job run. */ - private static boolean sourcesMatchIfPresent(Versions versions, JobRun jobRun) { - return ( ! versions.sourcePlatform.filter(version -> ! version.equals(versions.targetPlatform)).isPresent() - || versions.sourcePlatform.equals(jobRun.sourcePlatform())) - && ( ! versions.sourceApplication.filter(version -> ! version.equals(versions.targetApplication)).isPresent() - || versions.sourceApplication.equals(jobRun.sourceApplication())); - } - - private static boolean targetsMatch(Versions versions, JobRun jobRun) { - return versions.targetPlatform.equals(jobRun.platform()) && versions.targetApplication.equals(jobRun.application()); - } - // ---------- Change management o_O ---------- private boolean acceptNewApplicationVersion(Application application) { @@ -415,9 +438,10 @@ public class DeploymentTrigger { } private Change remainingChange(Application application) { - List<JobType> jobs = productionStepsOf(application).isEmpty() - ? jobsOf(testStepsOf(application)) - : jobsOf(productionStepsOf(application)); + DeploymentSteps steps = steps(application.deploymentSpec()); + List<JobType> jobs = steps.production().isEmpty() + ? steps.testJobs() + : steps.productionJobs(); Change change = application.change(); if (jobs.stream().allMatch(job -> isComplete(application.change().withoutApplication(), application, job))) @@ -436,61 +460,28 @@ public class DeploymentTrigger { */ private List<Job> testJobs(Application application, Versions versions, String reason, Instant availableSince) { List<Job> jobs = new ArrayList<>(); - for (JobType jobType : jobsOf(testStepsOf(application))) { + for (JobType jobType : steps(application.deploymentSpec()).testJobs()) { Optional<JobRun> completion = successOn(application, jobType, versions) - .filter(run -> sourcesMatchIfPresent(versions, run) || jobType == systemTest); - if ( ! completion.isPresent() && jobStateOf(application, jobType) == idle) + .filter(run -> versions.sourcesMatchIfPresent(run) || jobType == systemTest); + if (!completion.isPresent() && canTrigger(jobType, versions, application)) { jobs.add(deploymentJob(application, versions, application.change(), jobType, reason, availableSince)); + } } return jobs; } - private List<JobType> jobsOf(Collection<Step> steps) { - return steps.stream().flatMap(step -> step.zones().stream()).map(order::toJob).collect(toList()); - } - - private List<Step> testStepsOf(Application application) { - return application.deploymentSpec().steps().isEmpty() - ? singletonList(new DeploymentSpec.DeclaredZone(test)) - : application.deploymentSpec().steps().stream() - .filter(step -> step.deploysTo(test) || step.deploysTo(staging)) - .collect(toList()); - } - - private List<Step> productionStepsOf(Application application) { - return application.deploymentSpec().steps().stream() - .filter(step -> step.deploysTo(prod) || step.zones().isEmpty()) - .collect(toList()); - } - private Job deploymentJob(Application application, Versions versions, Change change, JobType jobType, String reason, Instant availableSince) { - boolean isRetry = application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::jobError) - .filter(JobError.outOfCapacity::equals).isPresent(); + boolean isRetry = application.deploymentJobs().statusOf(jobType) + .map(JobStatus::isOutOfCapacity) + .orElse(false); if (isRetry) reason += "; retrying on out of capacity"; - JobRun triggering = JobRun.triggering(versions.targetPlatform, versions.targetApplication, versions.sourcePlatform, versions.sourceApplication, reason, clock.instant()); + JobRun triggering = JobRun.triggering(versions.targetPlatform(), versions.targetApplication(), + versions.sourcePlatform(), versions.sourceApplication(), + reason, clock.instant()); return new Job(application, triggering, jobType, availableSince, isRetry, change.application().isPresent()); } - private Versions versions(Application application, Change change, Optional<Deployment> deployment) { - return new Versions(targetPlatform(application, change, deployment), - targetApplication(application, change, deployment), - deployment.map(Deployment::version), - deployment.map(Deployment::applicationVersion)); - } - - private Version targetPlatform(Application application, Change change, Optional<Deployment> deployment) { - return max(deployment.map(Deployment::version), change.platform()) - .orElse(application.oldestDeployedPlatform() - .orElse(controller.systemVersion())); - } - - private ApplicationVersion targetApplication(Application application, Change change, Optional<Deployment> deployment) { - return max(deployment.map(Deployment::applicationVersion), change.application()) - .orElse(application.oldestDeployedApplication() - .orElse(application.deploymentJobs().jobStatus().get(component).lastSuccess().get().application())); - } - // ---------- Data containers ---------- @@ -519,34 +510,5 @@ public class DeploymentTrigger { } - - private static class Versions { - - private final Version targetPlatform; - private final ApplicationVersion targetApplication; - private final Optional<Version> sourcePlatform; - private final Optional<ApplicationVersion> sourceApplication; - - private Versions(Version targetPlatform, ApplicationVersion targetApplication, Optional<Version> sourcePlatform, - Optional<ApplicationVersion> sourceApplication) { - this.targetPlatform = targetPlatform; - this.targetApplication = targetApplication; - this.sourcePlatform = sourcePlatform; - this.sourceApplication = sourceApplication; - } - - @Override - public String toString() { - return String.format("platform %s%s, application %s%s", - sourcePlatform.filter(source -> ! source.equals(targetPlatform)) - .map(source -> source + " -> ").orElse(""), - targetPlatform, - sourceApplication.filter(source -> ! source.equals(targetApplication)) - .map(source -> source.id() + " -> ").orElse(""), - targetApplication.id()); - } - - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java new file mode 100644 index 00000000000..74dffc1c4fd --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java @@ -0,0 +1,36 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.BuildService; + +/** + * Wraps a JobController as a BuildService. + * + * Shall be inlined when the {@link DelegatingBuildService} delegates all jobs to it. + * + * @author jonmv + */ +public class InternalBuildService implements BuildService { + + private final JobController jobs; + + public InternalBuildService(JobController jobs) { + this.jobs = jobs; + } + + @Override + public void trigger(BuildJob buildJob) { + + } + + @Override + public JobState stateOf(BuildJob buildJob) { + return null; + } + + @Override + public boolean builds(BuildJob buildJob) { + return false; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java new file mode 100644 index 00000000000..2d1040f7095 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -0,0 +1,93 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.LogStore; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; + +import java.util.List; + +/** + * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs. + * + * Keys are the {@link ApplicationId} of the real application, for which the deployment job is run, and the + * {@link ZoneId} of the real deployment to test. + * + * Although the deployment jobs are themselves applications, their IDs are not to be referenced. + * + * @author jonmv + */ +public class JobController { + + private final Controller controller; + private final LogStore logs; + + public JobController(Controller controller) { + this.controller = controller; + this.logs = null; + } + + +// GET: + /** Returns whether the given application has registered with this build service. */ + boolean builds(ApplicationId application) { + return false; + } + + /** Returns a list of all application which have registered. */ + List<ApplicationId> applications() { + return null; + } + + /** Returns all job types which have been run for the given application. */ + List<ZoneId> jobs(ApplicationId application) { + return null; + } + + /** Returns a list of meta information about all known runs of the given job type. */ + List<JobMeta> runs(ApplicationId application, ZoneId zone) { + return null; + } + + /** Returns the current status of the given job. */ + JobMeta status(JobId job) { + return null; + } + + /** Returns the details for the given job. */ + JobDetails details(JobId job) { + return null; + } + + +// POST: + /** Registers the given application, such that it may have deployment jobs run here. */ + void register(ApplicationId application) { + ; + } + + /** Orders a run of the given type, and returns the id of the created job. */ + JobId run(ApplicationId application, ZoneId zone) { + return null; + } + + +// PUT: + /** Stores the given details for the given job. */ + void store(JobDetails details, JobId job) { + ; + } + + +// DELETE: + /** Unregisters the given application, and deletes all associated data. */ + void unregister(ApplicationId application) { + ; + } + + /** Aborts the given job. */ + void abort(JobId job) { + ; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobDetails.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobDetails.java new file mode 100644 index 00000000000..3c787c8314f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobDetails.java @@ -0,0 +1,22 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.vespa.hosted.controller.api.ActivateResult; + +/** + * Contains details about a deployment job run. + * + * @author jonmv + */ +public class JobDetails { + + private final ActivateResult deploymentResult; + private final String convergenceLog; + private final String testLog; + + public JobDetails(ActivateResult deploymentResult, String convergenceLog, String testLog) { + this.deploymentResult = deploymentResult; + this.convergenceLog = convergenceLog; + this.testLog = testLog; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobId.java new file mode 100644 index 00000000000..541494a23fc --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobId.java @@ -0,0 +1,57 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; + +import java.util.Objects; + +/** + * Immutable ID of a job run by an {@link InternalBuildService}. + * + * @author jonmv + */ +public class JobId { + + private final ApplicationId application; + private final ZoneId zone; + private final long number; + + public JobId(ApplicationId application, ZoneId zone, long number) { + this.application = Objects.requireNonNull(application, "ApplicationId cannot be null!"); + this.zone = Objects.requireNonNull(zone, "ZoneId cannot be null!"); + if (number <= 0) throw new IllegalArgumentException("Build number must be a positive integer!"); + this.number = number; + } + + public ApplicationId application() { return application; } + public ZoneId zone() { return zone; } + public long number() { return number; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ( ! (o instanceof JobId)) return false; + + JobId id = (JobId) o; + + if (number != id.number) return false; + if ( ! application.equals(id.application)) return false; + return zone == id.zone; + } + + @Override + public int hashCode() { + int result = application.hashCode(); + result = 31 * result + zone.hashCode(); + result = 31 * result + (int) (number ^ (number >>> 32)); + return result; + } + + @Override + public String toString() { + return "Run " + number + " in " + zone + " for " + application; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMeta.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMeta.java new file mode 100644 index 00000000000..dde675402ce --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMeta.java @@ -0,0 +1,46 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import java.time.Instant; +import java.util.Optional; + +/** + * Contains state information for a deployment job run by an {@link InternalBuildService}. + * + * @author jonmv + */ +public class JobMeta { + + private final JobId id; + private final JobState state; + private final JobOutcome outcome; + private final Instant start; + private final Instant end; + + public JobMeta(JobId id, JobState state, JobOutcome outcome, Instant start, Instant end) { + this.id = id; + this.state = state; + this.outcome = outcome; + this.start = start; + this.end = end; + } + + public JobId id() { + return id; + } + + public JobState state() { + return state; + } + + public JobOutcome outcome() { + return outcome; + } + + public Instant start() { + return start; + } + + public Optional<Instant> end() { + return Optional.ofNullable(end); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobOutcome.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobOutcome.java new file mode 100644 index 00000000000..caecdcffb9b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobOutcome.java @@ -0,0 +1,31 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +/** + * Outcomes of jobs run by an {@link InternalBuildService}. + * + * @author jonmv + */ +public enum JobOutcome { + + /** Deployment of the real application was rejected due to missing capacity. */ + outOfCapacity, + + /** Deployment of the real application was rejected. */ + deploymentFailed, + + /** Convergence of the real application timed out. */ + convergenceFailed, + + /** Real application was deployed, but the tester application was not. */ + testError, + + /** Real application was deployed, but the tests failed. */ + testFailure, + + /** Deployment and tests completed with great success! */ + success, + + /** Job completed abnormally, due to user intervention or unexpected system error. */ + aborted + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobState.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobState.java new file mode 100644 index 00000000000..19e575efaf8 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobState.java @@ -0,0 +1,28 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +/** + * Status of jobs run by an {@link InternalBuildService}. + * + * @author jonmv + */ +public enum JobState { + + /** Job is not currently running, and may be started. */ + idle, + + /** Real application is deploying. */ + deploying, + + /** Real application is converging. */ + converging, + + /** Tester is starting up, but is not yet ready to serve its status. */ + initializing, + + /** Job is up and running normally. */ + running, + + /** Tests are complete, and results may be fetched. */ + finished + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java new file mode 100644 index 00000000000..bf58bac177c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -0,0 +1,114 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.JobStatus; + +import java.util.Optional; + +/** + * Source and target versions for an application. + * + * @author jvenstad + * @author mpolden + */ +public class Versions { + + private final Version targetPlatform; + private final ApplicationVersion targetApplication; + private final Optional<Version> sourcePlatform; + private final Optional<ApplicationVersion> sourceApplication; + + public Versions(Version targetPlatform, ApplicationVersion targetApplication, Optional<Version> sourcePlatform, + Optional<ApplicationVersion> sourceApplication) { + this.targetPlatform = targetPlatform; + this.targetApplication = targetApplication; + this.sourcePlatform = sourcePlatform; + this.sourceApplication = sourceApplication; + } + + /** Target platform version for this */ + public Version targetPlatform() { + return targetPlatform; + } + + /** Target application version for this */ + public ApplicationVersion targetApplication() { + return targetApplication; + } + + /** Source platform version for this */ + public Optional<Version> sourcePlatform() { + return sourcePlatform; + } + + /** Source application version for this */ + public Optional<ApplicationVersion> sourceApplication() { + return sourceApplication; + } + + /** Returns whether source versions are present and match those of the given job run */ + public boolean sourcesMatchIfPresent(JobStatus.JobRun jobRun) { + return (!sourcePlatform.filter(version -> !version.equals(targetPlatform)).isPresent() || + sourcePlatform.equals(jobRun.sourcePlatform())) && + (!sourceApplication.filter(version -> !version.equals(targetApplication)).isPresent() || + sourceApplication.equals(jobRun.sourceApplication())); + } + + public boolean targetsMatch(JobStatus.JobRun jobRun) { + return targetPlatform.equals(jobRun.platform()) && + targetApplication.equals(jobRun.application()); + } + + @Override + public String toString() { + return String.format("platform %s%s, application %s%s", + sourcePlatform.filter(source -> !source.equals(targetPlatform)) + .map(source -> source + " -> ").orElse(""), + targetPlatform, + sourceApplication.filter(source -> !source.equals(targetApplication)) + .map(source -> source.id() + " -> ").orElse(""), + targetApplication.id()); + } + + /** Create versions using change contained in application */ + public static Versions from(Application application, Version defaultPlatformVersion) { + return from(application.change(), application, Optional.empty(), defaultPlatformVersion); + } + + /** Create versions using given change and application */ + public static Versions from(Change change, Application application, Optional<Deployment> deployment, + Version defaultPlatformVersion) { + return new Versions(targetPlatform(application, change, deployment, defaultPlatformVersion), + targetApplication(application, change, deployment), + deployment.map(Deployment::version), + deployment.map(Deployment::applicationVersion)); + } + + private static Version targetPlatform(Application application, Change change, Optional<Deployment> deployment, + Version defaultVersion) { + return max(deployment.map(Deployment::version), change.platform()) + .orElse(application.oldestDeployedPlatform() + .orElse(defaultVersion)); + } + + private static ApplicationVersion targetApplication(Application application, Change change, + Optional<Deployment> deployment) { + return max(deployment.map(Deployment::applicationVersion), change.application()) + .orElse(application.oldestDeployedApplication() + .orElse(application.deploymentJobs().jobStatus().get(JobType.component) + .lastSuccess() + .get() + .application())); + } + + private static <T extends Comparable<T>> Optional<T> max(Optional<T> o1, Optional<T> o2) { + return ! o1.isPresent() ? o2 : ! o2.isPresent() ? o1 : o1.get().compareTo(o2.get()) >= 0 ? o1 : o2; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 1a6bce2dba9..8db7231c207 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -22,7 +22,7 @@ import java.util.logging.Level; * * When to file new issues, escalate inactive ones, etc., is handled by the enclosed OwnershipIssues. * - * @author jvenstad + * @author jonmv */ public class ApplicationOwnershipConfirmer extends Maintainer { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index d471e553bb9..91eda31d779 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -28,7 +28,7 @@ import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence * Maintenance job which files issues for tenants when they have jobs which fails continuously * and escalates issues which are not handled in a timely manner. * - * @author jvenstad + * @author jonmv */ public class DeploymentIssueReporter extends Maintainer { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index ac0d08f5105..8603698d498 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; @@ -38,16 +37,16 @@ public class SystemUpgrader extends Maintainer { return; } - deploy(SystemApplication.all(), target.get()); + deployAll(target.get(), SystemApplication.all()); } /** Deploy a list of system applications until they converge on the given version */ - private void deploy(List<SystemApplication> applications, Version target) { + private void deployAll(Version target, List<SystemApplication> applications) { for (List<ZoneId> zones : controller().zoneRegistry().upgradePolicy().asList()) { boolean converged = true; for (ZoneId zone : zones) { try { - converged &= deployInZone(zone, applications, target); + converged &= deployAll(target, applications, zone); } catch (UnreachableNodeRepositoryException e) { converged = false; log.log(Level.WARNING, e.getMessage() + ". Continuing to next parallel deployed zone"); @@ -62,8 +61,8 @@ public class SystemUpgrader extends Maintainer { } } - /** @return true if all applications have converged to the target version in the zone */ - private boolean deployInZone(ZoneId zone, List<SystemApplication> applications, Version target) { + /** Returns whether all applications have converged to the target version in zone */ + private boolean deployAll(Version target, List<SystemApplication> applications, ZoneId zone) { boolean converged = true; for (SystemApplication application : applications) { if (convergedOn(target, application.dependencies(), zone)) { @@ -76,7 +75,7 @@ public class SystemUpgrader extends Maintainer { /** Deploy application on given version idempotently */ private void deploy(Version target, SystemApplication application, ZoneId zone) { - if (!wantedVersion(zone, application.id(), target).equals(target)) { + if (!wantedVersion(zone, application, target).equals(target)) { log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone)); controller().applications().deploy(application, zone, target); } @@ -87,28 +86,28 @@ public class SystemUpgrader extends Maintainer { } private boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { - return currentVersion(zone, application.id(), target).equals(target); + return currentVersion(zone, application, target).equals(target); } - private Version wantedVersion(ZoneId zone, ApplicationId application, Version defaultVersion) { + private Version wantedVersion(ZoneId zone, SystemApplication application, Version defaultVersion) { return minVersion(zone, application, Node::wantedVersion).orElse(defaultVersion); } - private Version currentVersion(ZoneId zone, ApplicationId application, Version defaultVersion) { + private Version currentVersion(ZoneId zone, SystemApplication application, Version defaultVersion) { return minVersion(zone, application, Node::currentVersion).orElse(defaultVersion); } - private Optional<Version> minVersion(ZoneId zone, ApplicationId application, Function<Node, Version> versionField) { + private Optional<Version> minVersion(ZoneId zone, SystemApplication application, Function<Node, Version> versionField) { try { return controller().configServer() .nodeRepository() - .listOperational(zone, application) + .list(zone, application.id(), SystemApplication.activeStates()) .stream() .map(versionField) .min(Comparator.naturalOrder()); } catch (Exception e) { throw new UnreachableNodeRepositoryException(String.format("Failed to get version for %s in %s: %s", - application, zone, Exceptions.toMessageString(e))); + application.id(), zone, Exceptions.toMessageString(e))); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 96d252d3d1c..d804afdf98e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -13,6 +13,7 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; @@ -398,8 +399,8 @@ public class ApplicationSerializer { private Optional<JobStatus> jobStatusFromSlime(Inspector object) { // if the job type has since been removed, ignore it - Optional<DeploymentJobs.JobType> jobType = - DeploymentJobs.JobType.fromOptionalJobName(object.field(jobTypeField).asString()); + Optional<JobType> jobType = + JobType.fromOptionalJobName(object.field(jobTypeField).asString()); if (! jobType.isPresent()) return Optional.empty(); Optional<JobError> jobError = Optional.empty(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index edf68054481..d6c0bf23da6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -48,6 +48,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFact import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; @@ -355,8 +356,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Jobs sorted according to deployment spec List<JobStatus> jobStatus = controller.applications().deploymentTrigger() - .deploymentOrder() - .sortBy(application.deploymentSpec(), application.deploymentJobs().jobStatus().values()); + .steps(application.deploymentSpec()) + .sortBy(application.deploymentJobs().jobStatus().values()); Cursor deploymentsArray = object.setArray("deploymentJobs"); for (JobStatus job : jobStatus) { @@ -396,8 +397,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Deployments sorted according to deployment spec List<Deployment> deployments = controller.applications().deploymentTrigger() - .deploymentOrder() - .sortBy(application.deploymentSpec().zones(), application.deployments().values()); + .steps(application.deploymentSpec()) + .sortBy2(application.deployments().values()); Cursor instancesArray = object.setArray("instances"); for (Deployment deployment : deployments) { Cursor deploymentObject = instancesArray.addObject(); @@ -856,7 +857,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } return new DeploymentJobs.JobReport( ApplicationId.from(tenantName, applicationName, report.field("instance").asString()), - DeploymentJobs.JobType.fromJobName(report.field("jobName").asString()), + JobType.fromJobName(report.field("jobName").asString()), report.field("projectId").asLong(), report.field("buildNumber").asLong(), toSourceRevision(report.field("sourceRevision")), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java index c1a2c575fc2..26d0b1868ae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java @@ -10,7 +10,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.Path; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index fdfe20d004f..86950f9a61c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -130,7 +130,8 @@ public class VersionStatus { ListMap<Version, HostName> versions = new ListMap<>(); for (ZoneId zone : zones) { for (SystemApplication application : SystemApplication.all()) { - for (Node node : controller.configServer().nodeRepository().listOperational(zone, application.id())) { + for (Node node : controller.configServer().nodeRepository().list(zone, application.id(), + SystemApplication.activeStates())) { versions.put(node.currentVersion(), node.hostname()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index c24c8693688..621eb33bd66 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -16,22 +16,21 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; -import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -56,12 +55,12 @@ import java.util.stream.Collectors; import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.ControllerTester.buildJob; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionCorpUsEast1; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCorpUsEast1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -103,7 +102,7 @@ public class ControllerTest { assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); ApplicationVersion applicationVersion = tester.controller().applications().require(app1.id()).change().application().get(); - assertTrue("Application version has been set during deployment", applicationVersion != ApplicationVersion.unknown); + assertFalse("Application version has been set during deployment", applicationVersion.isUnknown()); assertStatus(JobStatus.initial(stagingTest) .withTriggering(version1, applicationVersion, Optional.empty(),"", tester.clock().instant()) .withCompletion(42, Optional.empty(), tester.clock().instant()), app1.id(), tester.controller()); @@ -151,6 +150,7 @@ public class ControllerTest { .withTriggering(version1, applicationVersion, productionCorpUsEast1.zone(main).map(tester.application(app1.id()).deployments()::get), "", tester.clock().instant()) .withCompletion(42, Optional.empty(), tester.clock().instant()), app1.id(), tester.controller()); + tester.clock().advance(Duration.ofHours(1)); // Stop retrying tester.jobCompletion(productionCorpUsEast1).application(app1).unsuccessful().submit(); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index cf2fa182d0a..d3b70ea07d8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; @@ -25,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRou import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.integration.MockMetricsService; @@ -122,7 +122,7 @@ public final class ControllerTester { }); } - public static BuildService.BuildJob buildJob(Application application, DeploymentJobs.JobType jobType) { + public static BuildService.BuildJob buildJob(Application application, JobType jobType) { return BuildService.BuildJob.of(application.id(), application.deploymentJobs().projectId().getAsLong(), jobType.jobName()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java index 821eb237530..bec909addec 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ArtifactRepositoryMock; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.SourceRevision; @@ -24,7 +25,7 @@ public class BuildJob { "master", "commit1"); public static final long defaultBuildNumber = 42; - private DeploymentJobs.JobType job; + private JobType job; private ApplicationId applicationId; private Optional<DeploymentJobs.JobError> jobError = Optional.empty(); private Optional<SourceRevision> sourceRevision = Optional.of(defaultSourceRevision); @@ -41,7 +42,7 @@ public class BuildJob { this.artifactRepository = artifactRepository; } - public BuildJob type(DeploymentJobs.JobType job) { + public BuildJob type(JobType job) { this.job = job; return this; } @@ -105,7 +106,7 @@ public class BuildJob { public BuildJob uploadArtifact(ApplicationPackage applicationPackage) { Objects.requireNonNull(job, "job cannot be null"); Objects.requireNonNull(applicationId, "applicationId cannot be null"); - if (job != DeploymentJobs.JobType.component) { + if (job != JobType.component) { throw new IllegalStateException(job + " cannot upload artifact"); } artifactRepository.put(applicationId, applicationPackage, applicationVersion()); @@ -114,7 +115,7 @@ public class BuildJob { /** Send report for this build job to the controller */ public void submit() { - if (job == DeploymentJobs.JobType.component && + if (job == JobType.component && !artifactRepository.contains(applicationId, applicationVersion())) { throw new IllegalStateException(job + " must upload artifact before reporting completion"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index ffc93f7b5ba..bae1f667566 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -12,13 +12,12 @@ import com.yahoo.vespa.hosted.controller.ArtifactRepositoryMock; import com.yahoo.vespa.hosted.controller.ConfigServerMock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; @@ -193,8 +192,8 @@ public class DeploymentTester { private void completeDeployment(Application application, ApplicationPackage applicationPackage, Optional<JobType> failOnJob, boolean includingProductionZones) { - DeploymentOrder order = new DeploymentOrder(controller()::system); - List<JobType> jobs = order.jobsFrom(applicationPackage.deploymentSpec()); + DeploymentSteps steps = controller().applications().deploymentTrigger().steps(applicationPackage.deploymentSpec()); + List<JobType> jobs = steps.jobs(); if ( ! includingProductionZones) jobs = jobs.stream().filter(job -> ! job.isProduction()).collect(Collectors.toList()); for (JobType job : jobs) { @@ -262,6 +261,10 @@ public class DeploymentTester { deployAndNotify(application, Optional.of(applicationPackage), success, job); } + public void deployAndNotify(Application application, boolean success, JobType job) { + deployAndNotify(application, Optional.empty(), success, job); + } + public void deployAndNotify(Application application, Optional<ApplicationPackage> applicationPackage, boolean success, JobType job) { if (success) { // Staging deploys twice, once with current version and once with new version @@ -296,8 +299,16 @@ public class DeploymentTester { .build(); } - public void assertRunning(ApplicationId id, JobType jobType) { - assertTrue(buildService().jobs().contains(BuildService.BuildJob.of(id, application(id).deploymentJobs().projectId().getAsLong(), jobType.jobName()))); + public void assertRunning(JobType job, ApplicationId application) { + assertTrue(String.format("Job %s for %s is running", job, application), isRunning(job, application)); + } + + public void assertNotRunning(JobType job, ApplicationId application) { + assertFalse(String.format("Job %s for %s is not running", job, application), isRunning(job, application)); + } + + private boolean isRunning(JobType job, ApplicationId application) { + return buildService().jobs().contains(ControllerTester.buildJob(application(application), job)); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index c26852a2153..033fddf3af8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -12,8 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; @@ -21,20 +20,19 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; -import java.util.Arrays; +import java.util.Optional; import java.util.function.Supplier; import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.ControllerTester.buildJob; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionEuWest1; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsCentral1; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionEuWest1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static java.util.Collections.singletonList; -import static java.util.Optional.empty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -43,7 +41,7 @@ import static org.junit.Assert.assertTrue; /** * @author bratseth * @author mpolden - * @author jvenstad + * @author jonmv */ public class DeploymentTriggerTest { @@ -62,8 +60,8 @@ public class DeploymentTriggerTest { // Deploy completely once tester.jobCompletion(component).application(app).uploadArtifact(applicationPackage).submit(); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.stagingTest); tester.deployAndNotify(app, applicationPackage, true, JobType.productionUsWest1); // New version is released @@ -74,11 +72,11 @@ public class DeploymentTriggerTest { tester.buildService().remove(buildJob(app, stagingTest)); tester.readyJobTrigger().maintain(); assertEquals("Retried dead job", 2, tester.buildService().jobs().size()); - tester.assertRunning(app.id(), stagingTest); + tester.assertRunning(stagingTest, app.id()); tester.deployAndNotify(app, applicationPackage, true, stagingTest); // system-test is now the only running job -- production jobs haven't started yet, since it is unfinished. - tester.assertRunning(app.id(), systemTest); + tester.assertRunning(systemTest, app.id()); assertEquals(1, tester.buildService().jobs().size()); // system-test fails and is retried @@ -86,7 +84,7 @@ public class DeploymentTriggerTest { assertEquals("Job is retried on failure", 1, tester.buildService().jobs().size()); tester.deployAndNotify(app, applicationPackage, true, JobType.systemTest); - tester.assertRunning(app.id(), productionUsWest1); + tester.assertRunning(productionUsWest1, app.id()); } @Test @@ -145,13 +143,13 @@ public class DeploymentTriggerTest { // 30 seconds later, the first jobs may trigger. assertEquals(1, mockBuildService.jobs().size()); - tester.assertRunning(application.id(), productionUsWest1); + tester.assertRunning(productionUsWest1, application.id()); // 3 minutes pass, delayed trigger does nothing as us-west-1 is still in progress tester.clock().advance(Duration.ofMinutes(3)); tester.deploymentTrigger().triggerReadyJobs(); assertEquals(1, mockBuildService.jobs().size()); - tester.assertRunning(application.id(), productionUsWest1); + tester.assertRunning(productionUsWest1, application.id()); // us-west-1 completes tester.deployAndNotify(application, applicationPackage, true, productionUsWest1); @@ -202,8 +200,8 @@ public class DeploymentTriggerTest { // Deploys in two regions in parallel assertEquals(2, tester.buildService().jobs().size()); - tester.assertRunning(application.id(), productionUsEast3); - tester.assertRunning(application.id(), productionUsWest1); + tester.assertRunning(productionUsEast3, application.id()); + tester.assertRunning(productionUsWest1, application.id()); tester.deploy(JobType.productionUsWest1, application, applicationPackage, false); tester.jobCompletion(JobType.productionUsWest1).application(application).submit(); @@ -230,17 +228,17 @@ public class DeploymentTriggerTest { tester.jobCompletion(component).application(app).uploadArtifact(applicationPackage).submit(); // Test environments pass - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.stagingTest); // Last declared job completes first - tester.deploy(DeploymentJobs.JobType.productionUsWest1, app, applicationPackage); - tester.jobCompletion(DeploymentJobs.JobType.productionUsWest1).application(app).submit(); + tester.deploy(JobType.productionUsWest1, app, applicationPackage); + tester.jobCompletion(JobType.productionUsWest1).application(app).submit(); assertTrue("Change is present as not all jobs are complete", tester.applications().require(app.id()).change().isPresent()); // All jobs complete - tester.deploy(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage); + tester.deploy(JobType.productionUsEast3, app, applicationPackage); tester.jobCompletion(JobType.productionUsEast3).application(app).submit(); assertFalse("Change has been deployed", tester.applications().require(app.id()).change().isPresent()); @@ -405,8 +403,8 @@ public class DeploymentTriggerTest { }); assertEquals(0, tester.buildService().jobs().size()); readyJobsTrigger.run(); - tester.assertRunning(app.id(), systemTest); - tester.assertRunning(app.id(), stagingTest); + tester.assertRunning(systemTest, app.id()); + tester.assertRunning(stagingTest, app.id()); } @Test @@ -426,8 +424,8 @@ public class DeploymentTriggerTest { tester.completeDeploymentWithError(application, applicationPackage, BuildJob.defaultBuildNumber + 1, productionUsCentral1); // deployAndNotify doesn't actually deploy if the job fails, so we need to do that manually. - tester.deployAndNotify(application, empty(), false, productionUsCentral1); - tester.deploy(productionUsCentral1, application, empty(), false); + tester.deployAndNotify(application, false, productionUsCentral1); + tester.deploy(productionUsCentral1, application, Optional.empty(), false); ApplicationVersion appVersion1 = ApplicationVersion.from(BuildJob.defaultSourceRevision, BuildJob.defaultBuildNumber + 1); assertEquals(appVersion1, app.get().deployments().get(ZoneId.from("prod.us-central-1")).applicationVersion()); @@ -444,18 +442,18 @@ public class DeploymentTriggerTest { Version version1 = new Version("6.2"); tester.upgradeSystem(version1); tester.jobCompletion(productionUsCentral1).application(application).unsuccessful().submit(); - tester.deployAndNotify(application, empty(), true, systemTest); - tester.deployAndNotify(application, empty(), true, stagingTest); - tester.deployAndNotify(application, empty(), false, productionUsCentral1); + tester.deployAndNotify(application, true, systemTest); + tester.deployAndNotify(application, true, stagingTest); + tester.deployAndNotify(application, false, productionUsCentral1); // The last job has a different target, and the tests need to run again. // These may now start, since the first job has been triggered once, and thus is verified already. - tester.deployAndNotify(application, empty(), true, systemTest); - tester.deployAndNotify(application, empty(), true, stagingTest); + tester.deployAndNotify(application, true, systemTest); + tester.deployAndNotify(application, true, stagingTest); // Finally, the two production jobs complete, in order. - tester.deployAndNotify(application, empty(), true, productionUsCentral1); - tester.deployAndNotify(application, empty(), true, productionEuWest1); + tester.deployAndNotify(application, true, productionUsCentral1); + tester.deployAndNotify(application, true, productionEuWest1); assertEquals(appVersion1, app.get().deployments().get(ZoneId.from("prod.us-central-1")).applicationVersion()); } @@ -500,22 +498,26 @@ public class DeploymentTriggerTest { tester.deployAndNotify(application, applicationPackage, true, systemTest); tester.deployAndNotify(application, applicationPackage, true, stagingTest); + tester.assertRunning(productionUsCentral1, application.id()); assertEquals(v2, app.get().deployments().get(productionUsCentral1.zone(main).get()).version()); - assertEquals((Long) 42L, app.get().deployments().get(productionUsCentral1.zone(main).get()).applicationVersion().buildNumber().get()); + assertEquals(Long.valueOf(42L), app.get().deployments().get(productionUsCentral1.zone(main).get()).applicationVersion().buildNumber().get()); assertNotEquals(triggered, app.get().deploymentJobs().jobStatus().get(productionUsCentral1).lastTriggered().get().at()); // Change has a higher application version than what is deployed -- deployment should trigger. tester.deployAndNotify(application, applicationPackage, false, productionUsCentral1); tester.deploy(productionUsCentral1, application, applicationPackage); assertEquals(v2, app.get().deployments().get(productionUsCentral1.zone(main).get()).version()); - assertEquals((Long) 43L, app.get().deployments().get(productionUsCentral1.zone(main).get()).applicationVersion().buildNumber().get()); + assertEquals(Long.valueOf(43), app.get().deployments().get(productionUsCentral1.zone(main).get()).applicationVersion().buildNumber().get()); // Change is again strictly dominated, and us-central-1 is skipped, even though it is still failing. - tester.deployAndNotify(application, applicationPackage, false, productionUsCentral1); + tester.clock().advance(Duration.ofHours(2).plus(Duration.ofSeconds(1))); // Enough time for retry + tester.readyJobTrigger().maintain(); + // Failing job is not retried as change has been deployed + tester.assertNotRunning(productionUsCentral1, application.id()); // Last job has a different deployment target, so tests need to run again. - tester.deployAndNotify(application, empty(), true, systemTest); - tester.deployAndNotify(application, empty(), true, stagingTest); + tester.deployAndNotify(application, true, systemTest); + tester.deployAndNotify(application, true, stagingTest); tester.deployAndNotify(application, applicationPackage, true, productionEuWest1); assertFalse(app.get().change().isPresent()); assertFalse(app.get().deploymentJobs().jobStatus().get(productionUsCentral1).isSuccess()); @@ -538,8 +540,8 @@ public class DeploymentTriggerTest { Version v1 = new Version("6.1"); Version v2 = new Version("6.2"); tester.upgradeSystem(v2); - tester.deployAndNotify(application, empty(), true, systemTest); - tester.deployAndNotify(application, empty(), true, stagingTest); + tester.deployAndNotify(application, true, systemTest); + tester.deployAndNotify(application, true, stagingTest); tester.deploymentTrigger().cancelChange(application.id(), true); tester.deploy(productionEuWest1, application, applicationPackage); assertEquals(v2, app.get().deployments().get(productionEuWest1.zone(main).get()).version()); @@ -550,8 +552,8 @@ public class DeploymentTriggerTest { Version firstTested = app.get().deploymentJobs().jobStatus().get(systemTest).lastTriggered().get().platform(); assertEquals(firstTested, app.get().deploymentJobs().jobStatus().get(stagingTest).lastTriggered().get().platform()); - tester.deployAndNotify(application, empty(), true, systemTest); - tester.deployAndNotify(application, empty(), true, stagingTest); + tester.deployAndNotify(application, true, systemTest); + tester.deployAndNotify(application, true, stagingTest); // Tests are not re-triggered, because the jobs they were run for has not yet been triggered with the tested versions. assertEquals(firstTested, app.get().deploymentJobs().jobStatus().get(systemTest).lastTriggered().get().platform()); @@ -565,14 +567,14 @@ public class DeploymentTriggerTest { // New upgrade is already tested for one of the jobs, which has now been triggered, and tests may run for the other job. assertNotEquals(firstTested, app.get().deploymentJobs().jobStatus().get(systemTest).lastTriggered().get().platform()); assertNotEquals(firstTested, app.get().deploymentJobs().jobStatus().get(stagingTest).lastTriggered().get().platform()); - tester.deployAndNotify(application, empty(), true, systemTest); - tester.deployAndNotify(application, empty(), true, stagingTest); + tester.deployAndNotify(application, true, systemTest); + tester.deployAndNotify(application, true, stagingTest); // Both jobs fail again, and must be re-triggered -- this is ok, as they are both already triggered on their current targets. - tester.deployAndNotify(application, empty(), false, productionEuWest1); - tester.deployAndNotify(application, empty(), false, productionUsEast3); - tester.deployAndNotify(application, empty(), true, productionUsEast3); - tester.deployAndNotify(application, empty(), true, productionEuWest1); + tester.deployAndNotify(application, false, productionEuWest1); + tester.deployAndNotify(application, false, productionUsEast3); + tester.deployAndNotify(application, true, productionUsEast3); + tester.deployAndNotify(application, true, productionEuWest1); assertFalse(app.get().change().isPresent()); assertEquals(43, app.get().deploymentJobs().jobStatus().get(productionEuWest1).lastSuccess().get().application().buildNumber().get().longValue()); assertEquals(43, app.get().deploymentJobs().jobStatus().get(productionUsEast3).lastSuccess().get().application().buildNumber().get().longValue()); @@ -595,28 +597,86 @@ public class DeploymentTriggerTest { Version v1 = new Version("6.1"); Version v2 = new Version("6.2"); tester.upgradeSystem(v2); - tester.deployAndNotify(application, empty(), true, systemTest); - tester.deployAndNotify(application, empty(), true, stagingTest); - tester.deployAndNotify(application, empty(), true, productionUsCentral1); - tester.deployAndNotify(application, empty(), true, productionEuWest1); - tester.deployAndNotify(application, empty(), false, productionUsEast3); + tester.deployAndNotify(application, true, systemTest); + tester.deployAndNotify(application, true, stagingTest); + tester.deployAndNotify(application, true, productionUsCentral1); + tester.deployAndNotify(application, true, productionEuWest1); + tester.deployAndNotify(application, false, productionUsEast3); assertEquals(v2, app.get().deployments().get(ZoneId.from("prod", "us-central-1")).version()); assertEquals(v2, app.get().deployments().get(ZoneId.from("prod", "eu-west-1")).version()); assertEquals(v1, app.get().deployments().get(ZoneId.from("prod", "us-east-3")).version()); Version v3 = new Version("6.3"); tester.upgradeSystem(v3); - tester.deployAndNotify(application, empty(), false, productionUsEast3); + tester.deployAndNotify(application, false, productionUsEast3); // See that sources for staging are: first v2, then v1. - tester.deployAndNotify(application, empty(), true, systemTest); - tester.deployAndNotify(application, empty(), true, stagingTest); + tester.deployAndNotify(application, true, systemTest); + tester.deployAndNotify(application, true, stagingTest); assertEquals(v2, app.get().deploymentJobs().jobStatus().get(stagingTest).lastSuccess().get().sourcePlatform().get()); - tester.deployAndNotify(application, empty(), true, productionUsCentral1); + tester.deployAndNotify(application, true, productionUsCentral1); assertEquals(v1, app.get().deploymentJobs().jobStatus().get(stagingTest).lastTriggered().get().sourcePlatform().get()); - tester.deployAndNotify(application, empty(), true, stagingTest); - tester.deployAndNotify(application, empty(), true, productionEuWest1); - tester.deployAndNotify(application, empty(), true, productionUsEast3); + tester.deployAndNotify(application, true, stagingTest); + tester.deployAndNotify(application, true, productionEuWest1); + tester.deployAndNotify(application, true, productionUsEast3); + } + + @Test + public void retriesFailingJobs() { + DeploymentTester tester = new DeploymentTester(); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-central-1") + .build(); + + // Deploy completely on default application and platform versions + tester.deployCompletely(application, applicationPackage); + + // New application change is deployed and fails in system-test for a while + tester.jobCompletion(component).application(application).nextBuildNumber().uploadArtifact(applicationPackage).submit(); + tester.deployAndNotify(application, false, systemTest); + tester.deployAndNotify(application, true, stagingTest); + + // Retries immediately in the first minute after failing + tester.clock().advance(Duration.ofSeconds(59)); + tester.jobCompletion(systemTest).application(application).unsuccessful().submit(); + tester.readyJobTrigger().maintain(); + tester.assertRunning(systemTest, application.id()); + + // Stops immediate retry after failing for 1 minute + tester.clock().advance(Duration.ofSeconds(1)); + tester.jobCompletion(systemTest).application(application).unsuccessful().submit(); + tester.readyJobTrigger().maintain(); + tester.assertNotRunning(systemTest, application.id()); + + // Retries after 10 minutes since previous completion as we failed within the last hour + tester.clock().advance(Duration.ofMinutes(10).plus(Duration.ofSeconds(1))); + tester.readyJobTrigger().maintain(); + tester.assertRunning(systemTest, application.id()); + + // Retries less frequently after 1 hour of failure + tester.clock().advance(Duration.ofMinutes(50)); + tester.jobCompletion(systemTest).application(application).unsuccessful().submit(); + tester.readyJobTrigger().maintain(); + tester.assertNotRunning(systemTest, application.id()); + + // Retries after two hours pass since last completion + tester.clock().advance(Duration.ofHours(2).plus(Duration.ofSeconds(1))); + tester.readyJobTrigger().maintain(); + tester.assertRunning(systemTest, application.id()); + + // Still fails and is not retried + tester.jobCompletion(systemTest).application(application).unsuccessful().submit(); + tester.readyJobTrigger().maintain(); + tester.assertNotRunning(systemTest, application.id()); + + // Another application change is deployed and fixes system-test. Change is triggered immediately as target changes + tester.jobCompletion(component).application(application).nextBuildNumber(2).uploadArtifact(applicationPackage).submit(); + tester.deployAndNotify(application, true, systemTest); + tester.deployAndNotify(application, true, stagingTest); + tester.deployAndNotify(application, true, productionUsCentral1); + assertTrue("Deployment completed", tester.buildService().jobs().isEmpty()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java index b5941c441e2..703d65c8f9d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** - * @author jvenstad + * @author jonmv */ public class ApplicationOwnershipConfirmerTest { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index 2abd01927a1..a1aef28bd63 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -19,10 +19,10 @@ import java.time.Duration; import java.util.HashMap; import java.util.Map; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionCorpUsEast1; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCorpUsEast1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.maxFailureAge; import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.maxInactivity; import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.upgradeGracePeriod; @@ -31,7 +31,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** - * @author jvenstad + * @author jonmv */ public class DeploymentIssueReporterTest { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java index af7325950e4..1bedb29ec97 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java @@ -18,8 +18,8 @@ import org.junit.Test; import java.time.Duration; import java.util.Optional; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index 981b8c8c52a..307007e394a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -8,8 +8,8 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; @@ -19,9 +19,9 @@ import java.nio.file.Paths; import java.time.Duration; import java.util.Collections; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -44,9 +44,9 @@ public class FailureRedeployerTest { Application app = tester.createApplication("app1", "tenant1", 1, 11L); tester.jobCompletion(component).application(app).uploadArtifact(applicationPackage).submit(); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); + tester.deployAndNotify(app, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.productionUsEast3); // New version is released version = Version.fromString("5.1"); @@ -56,12 +56,12 @@ public class FailureRedeployerTest { tester.readyJobTrigger().maintain(); // Test environments pass - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.stagingTest); // Production job fails and is retried tester.clock().advance(Duration.ofSeconds(1)); // Advance time so that we can detect jobs in progress - tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3); + tester.deployAndNotify(app, applicationPackage, false, JobType.productionUsEast3); assertEquals("Production job is retried", 1, tester.buildService().jobs().size()); assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).change().platform().get()); @@ -69,24 +69,24 @@ public class FailureRedeployerTest { version = Version.fromString("5.2"); tester.upgradeSystem(version); tester.upgrader().maintain(); - tester.jobCompletion(DeploymentJobs.JobType.productionUsEast3).application(app).unsuccessful().submit(); + tester.jobCompletion(JobType.productionUsEast3).application(app).unsuccessful().submit(); assertEquals("Application starts upgrading to new version", 2, tester.buildService().jobs().size()); assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).change().platform().get()); // Failure re-deployer did not retry failing job for prod.us-east-3, since it no longer had an available change assertFalse("Job is not retried", tester.buildService().jobs().stream() - .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.jobName()))); + .anyMatch(j -> j.jobName().equals(JobType.productionUsEast3.jobName()))); // Test environments pass - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.stagingTest); // Production job fails again, and is retried - tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3); + tester.deployAndNotify(app, applicationPackage, false, JobType.productionUsEast3); assertEquals("Job is retried", Collections.singletonList(ControllerTester.buildJob(app, productionUsEast3)), tester.buildService().jobs()); // Production job finally succeeds - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); + tester.deployAndNotify(app, applicationPackage, true, JobType.productionUsEast3); assertTrue("All jobs consumed", tester.buildService().jobs().isEmpty()); assertFalse("No failures", tester.application(app.id()).deploymentJobs().hasFailures()); } @@ -103,9 +103,9 @@ public class FailureRedeployerTest { Application app = tester.createApplication("app1", "tenant1", 1, 11L); tester.jobCompletion(component).application(app).uploadArtifact(applicationPackage).submit(); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); + tester.deployAndNotify(app, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.productionUsEast3); // New version is released version = Version.fromString("5.1"); @@ -116,7 +116,7 @@ public class FailureRedeployerTest { assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).change().platform().get()); // system-test fails and is left with a retry - tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, false, JobType.systemTest); // Another version is released version = Version.fromString("5.2"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index af61af6da52..e189a9243db 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -31,10 +31,10 @@ import java.time.Instant; import java.time.ZoneId; import java.util.Map; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 0b55d13a5ad..4483122d554 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -5,9 +5,9 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -46,7 +46,7 @@ public class OutstandingChangeDeployerTest { assertEquals(Change.of(version), tester.application("app1").change()); assertFalse(tester.application("app1").outstandingChange().isPresent()); - tester.jobCompletion(DeploymentJobs.JobType.component) + tester.jobCompletion(JobType.component) .application(tester.application("app1")) .sourceRevision(new SourceRevision("repository1","master", "cafed00d")) .nextBuildNumber() @@ -63,9 +63,9 @@ public class OutstandingChangeDeployerTest { assertEquals("No effect as job is in progress", 2, tester.buildService().jobs().size()); assertEquals("1.0.43-cafed00d", app.outstandingChange().application().get().id()); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsWest1); + tester.deployAndNotify(app, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.productionUsWest1); assertEquals("Upgrade done", 0, tester.buildService().jobs().size()); deployer.maintain(); @@ -75,8 +75,8 @@ public class OutstandingChangeDeployerTest { List<BuildService.BuildJob> jobs = tester.buildService().jobs(); assertEquals(2, jobs.size()); assertEquals(11, jobs.get(0).projectId()); - tester.assertRunning(app.id(), DeploymentJobs.JobType.systemTest); - tester.assertRunning(app.id(), DeploymentJobs.JobType.stagingTest); + tester.assertRunning(JobType.systemTest, app.id()); + tester.assertRunning(JobType.stagingTest, app.id()); assertFalse(tester.application("app1").outstandingChange().isPresent()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 4b563ed203d..5bfda19863f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -2,14 +2,12 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import org.junit.Ignore; import org.junit.Test; import java.util.Arrays; @@ -192,7 +190,7 @@ public class SystemUpgraderTest { private void completeUpgrade(SystemApplication application, Version version, ZoneId... zones) { assertWantedVersion(application, version, zones); for (ZoneId zone : zones) { - for (Node node : nodeRepository().listOperational(zone, application.id())) { + for (Node node : nodeRepository().list(zone, application.id(), SystemApplication.activeStates())) { nodeRepository().add(zone, new Node(node.hostname(), node.state(), node.type(), node.owner(), node.wantedVersion(), node.wantedVersion())); } @@ -215,25 +213,25 @@ public class SystemUpgraderTest { } private void assertWantedVersion(SystemApplication application, Version version, ZoneId... zones) { - assertVersion(application.id(), version, Node::wantedVersion, zones); + assertVersion(application, version, Node::wantedVersion, zones); } private void assertCurrentVersion(SystemApplication application, Version version, ZoneId... zones) { - assertVersion(application.id(), version, Node::currentVersion, zones); + assertVersion(application, version, Node::currentVersion, zones); } private void assertWantedVersion(List<SystemApplication> applications, Version version, ZoneId... zones) { - applications.forEach(application -> assertVersion(application.id(), version, Node::wantedVersion, zones)); + applications.forEach(application -> assertVersion(application, version, Node::wantedVersion, zones)); } private void assertCurrentVersion(List<SystemApplication> applications, Version version, ZoneId... zones) { - applications.forEach(application -> assertVersion(application.id(), version, Node::currentVersion, zones)); + applications.forEach(application -> assertVersion(application, version, Node::currentVersion, zones)); } - private void assertVersion(ApplicationId application, Version version, Function<Node, Version> versionField, + private void assertVersion(SystemApplication application, Version version, Function<Node, Version> versionField, ZoneId... zones) { for (ZoneId zone : zones) { - for (Node node : nodeRepository().listOperational(zone, application)) { + for (Node node : nodeRepository().list(zone, application.id(), SystemApplication.activeStates())) { assertEquals(application + " version", version, versionField.apply(node)); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 5d6fb76cacf..781933eda64 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -7,10 +7,10 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -19,12 +19,12 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsCentral1; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -220,7 +220,7 @@ public class UpgraderTest { tester.completeUpgradeWithError(default3, version54, "default", stagingTest); - tester.completeUpgradeWithError(default4, version54, "default", DeploymentJobs.JobType.productionUsWest1); + tester.completeUpgradeWithError(default4, version54, "default", JobType.productionUsWest1); // State: Default applications started upgrading to 5.5 tester.clock().advance(Duration.ofHours(1)); tester.upgrader().maintain(); @@ -229,13 +229,15 @@ public class UpgraderTest { tester.completeUpgradeWithError(default0, version55, "default", stagingTest); tester.completeUpgradeWithError(default1, version55, "default", stagingTest); tester.completeUpgradeWithError(default2, version55, "default", stagingTest); - tester.completeUpgradeWithError(default3, version55, "default", DeploymentJobs.JobType.productionUsWest1); + tester.clock().advance(Duration.ofHours(2).plus(Duration.ofSeconds(1))); // Retry failing job for default3 + tester.readyJobTrigger().maintain(); + tester.completeUpgradeWithError(default3, version55, "default", JobType.productionUsWest1); tester.upgradeSystem(version55); assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); // Finish running job, without retry. tester.clock().advance(Duration.ofHours(1)); - tester.jobCompletion(DeploymentJobs.JobType.productionUsWest1).application(default3).unsuccessful().submit(); + tester.jobCompletion(JobType.productionUsWest1).application(default3).unsuccessful().submit(); tester.upgrader().maintain(); tester.buildService().clear(); @@ -801,7 +803,7 @@ public class UpgraderTest { // 5th app never reports back and has a dead job, but no ongoing change Application deadLocked = tester.applications().require(default4.id()); - tester.assertRunning(deadLocked.id(), systemTest); + tester.assertRunning(systemTest, deadLocked.id()); assertFalse("No change present", deadLocked.change().isPresent()); // 4 out of 5 applications are repaired and confidence is restored diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 656c377b84b..95a785ca0df 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -10,6 +10,7 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; @@ -78,13 +79,13 @@ public class ApplicationSerializerTest { OptionalLong projectId = OptionalLong.of(123L); List<JobStatus> statusList = new ArrayList<>(); - statusList.add(JobStatus.initial(DeploymentJobs.JobType.systemTest) + statusList.add(JobStatus.initial(JobType.systemTest) .withTriggering(Version.fromString("5.6.7"), ApplicationVersion.unknown, empty(), "Test", Instant.ofEpochMilli(7)) .withCompletion(30, empty(), Instant.ofEpochMilli(8))); - statusList.add(JobStatus.initial(DeploymentJobs.JobType.stagingTest) + statusList.add(JobStatus.initial(JobType.stagingTest) .withTriggering(Version.fromString("5.6.6"), ApplicationVersion.unknown, empty(), "Test 2", Instant.ofEpochMilli(5)) .withCompletion(11, Optional.of(JobError.unknown), Instant.ofEpochMilli(6))); - statusList.add(JobStatus.initial(DeploymentJobs.JobType.from(main, zone1).get()) + statusList.add(JobStatus.initial(JobType.from(main, zone1).get()) .withTriggering(Version.fromString("5.6.6"), ApplicationVersion.unknown, deployments.stream().findFirst(), "Test 3", Instant.ofEpochMilli(6)) .withCompletion(11, empty(), Instant.ofEpochMilli(7))); @@ -119,10 +120,10 @@ public class ApplicationSerializerTest { assertEquals(original.deploymentJobs().projectId(), serialized.deploymentJobs().projectId()); assertEquals(original.deploymentJobs().jobStatus().size(), serialized.deploymentJobs().jobStatus().size()); - assertEquals( original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest), - serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest)); - assertEquals( original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest), - serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest)); + assertEquals( original.deploymentJobs().jobStatus().get(JobType.systemTest), + serialized.deploymentJobs().jobStatus().get(JobType.systemTest)); + assertEquals( original.deploymentJobs().jobStatus().get(JobType.stagingTest), + serialized.deploymentJobs().jobStatus().get(JobType.stagingTest)); assertEquals(original.outstandingChange(), serialized.outstandingChange()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 51437be2832..6d88cf79241 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -12,14 +12,12 @@ import com.yahoo.vespa.hosted.controller.ArtifactRepositoryMock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TestIdentities; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; -import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; -import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; -import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.HostedAthenzIdentities; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -38,7 +36,7 @@ import java.io.IOException; import java.time.Duration; import java.util.Optional; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; import static org.junit.Assert.assertFalse; /** @@ -92,7 +90,7 @@ public class ContainerControllerTester { } /** Notify the controller about a job completing */ - public BuildJob jobCompletion(DeploymentJobs.JobType job) { + public BuildJob jobCompletion(JobType job) { return new BuildJob(this::notifyJobCompletion, artifactRepository()).type(job); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 545ee529635..c748d32ef0b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.Applicat import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockOrganization; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; @@ -194,7 +195,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value())); // (Necessary but not provided in this API) // Trigger deployment from completion of component job - controllerTester.jobCompletion(DeploymentJobs.JobType.component) + controllerTester.jobCompletion(JobType.component) .application(id) .projectId(screwdriverProjectId) .uploadArtifact(applicationPackage) @@ -209,7 +210,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .screwdriverIdentity(SCREWDRIVER_ID), "Deactivated tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default"); // Called through the separate screwdriver/v1 API - controllerTester.jobCompletion(DeploymentJobs.JobType.systemTest) + controllerTester.jobCompletion(JobType.systemTest) .application(id) .projectId(screwdriverProjectId) .submit(); @@ -222,7 +223,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), "Deactivated tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default"); - controllerTester.jobCompletion(DeploymentJobs.JobType.stagingTest) + controllerTester.jobCompletion(JobType.stagingTest) .application(id) .projectId(screwdriverProjectId) .submit(); @@ -232,7 +233,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(createApplicationDeployData(applicationPackage, false)) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); - controllerTester.jobCompletion(DeploymentJobs.JobType.productionCorpUsEast1) + controllerTester.jobCompletion(JobType.productionCorpUsEast1) .application(id) .projectId(screwdriverProjectId) .unsuccessful() @@ -492,7 +493,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); - controllerTester.jobCompletion(DeploymentJobs.JobType.productionUsEast3) + controllerTester.jobCompletion(JobType.productionUsEast3) .application(id) .projectId(projectId) .submit(); @@ -511,7 +512,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); - controllerTester.jobCompletion(DeploymentJobs.JobType.productionUsWest1) + controllerTester.jobCompletion(JobType.productionUsWest1) .application(id) .projectId(projectId) .submit(); @@ -521,7 +522,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); - controllerTester.jobCompletion(DeploymentJobs.JobType.productionUsEast3) + controllerTester.jobCompletion(JobType.productionUsEast3) .application(id) .projectId(projectId) .submit(); @@ -830,7 +831,7 @@ public class ApplicationApiTest extends ControllerContainerTest { controllerTester.authorize(ATHENZ_TENANT_DOMAIN, screwdriverId, ApplicationAction.deploy, application); // Allow systemtest to succeed by notifying completion of system test - controllerTester.jobCompletion(DeploymentJobs.JobType.component) + controllerTester.jobCompletion(JobType.component) .application(application.id()) .projectId(screwdriverProjectId) .uploadArtifact(applicationPackage) @@ -859,27 +860,27 @@ public class ApplicationApiTest extends ControllerContainerTest { BuildJob job = new BuildJob(report -> notifyCompletion(report, tester), tester.artifactRepository()) .application(app) .projectId(projectId); - job.type(DeploymentJobs.JobType.component).uploadArtifact(applicationPackage).submit(); + job.type(JobType.component).uploadArtifact(applicationPackage).submit(); tester.deploy(app, applicationPackage, TEST_ZONE, projectId); - job.type(DeploymentJobs.JobType.systemTest).submit(); + job.type(JobType.systemTest).submit(); // Notifying about unknown job fails Request request = request("/application/v4/tenant/tenant1/application/application1/jobreport", POST) - .data(asJson(job.type(DeploymentJobs.JobType.productionUsEast3).report())) + .data(asJson(job.type(JobType.productionUsEast3).report())) .userIdentity(HOSTED_VESPA_OPERATOR) .get(); tester.containerTester().assertResponse(request, new File("jobreport-unexpected-completion.json"), 400); // ... and assert it was recorded JobStatus recordedStatus = - tester.controller().applications().get(app.id()).get().deploymentJobs().jobStatus().get(DeploymentJobs.JobType.component); + tester.controller().applications().get(app.id()).get().deploymentJobs().jobStatus().get(JobType.component); assertNotNull("Status was recorded", recordedStatus); assertTrue(recordedStatus.isSuccess()); assertEquals(vespaVersion, recordedStatus.lastCompleted().get().platform()); recordedStatus = - tester.controller().applications().get(app.id()).get().deploymentJobs().jobStatus().get(DeploymentJobs.JobType.productionApNortheast2); + tester.controller().applications().get(app.id()).get().deploymentJobs().jobStatus().get(JobType.productionApNortheast2); assertNull("Status of never-triggered jobs is empty", recordedStatus); Response response; @@ -906,19 +907,19 @@ public class ApplicationApiTest extends ControllerContainerTest { BuildJob job = new BuildJob(report -> notifyCompletion(report, tester), tester.artifactRepository()) .application(app) .projectId(projectId); - job.type(DeploymentJobs.JobType.component).uploadArtifact(applicationPackage).submit(); + job.type(JobType.component).uploadArtifact(applicationPackage).submit(); tester.deploy(app, applicationPackage, TEST_ZONE, projectId); - job.type(DeploymentJobs.JobType.systemTest).submit(); + job.type(JobType.systemTest).submit(); tester.deploy(app, applicationPackage, STAGING_ZONE, projectId); - job.type(DeploymentJobs.JobType.stagingTest).error(DeploymentJobs.JobError.outOfCapacity).submit(); + job.type(JobType.stagingTest).error(DeploymentJobs.JobError.outOfCapacity).submit(); // Appropriate error is recorded JobStatus jobStatus = tester.controller().applications().get(app.id()) .get() .deploymentJobs() .jobStatus() - .get(DeploymentJobs.JobType.stagingTest); + .get(JobType.stagingTest); assertFalse(jobStatus.isSuccess()); assertEquals(DeploymentJobs.JobError.outOfCapacity, jobStatus.jobError().get()); } @@ -1062,7 +1063,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Trigger application change controllerTester.artifactRepository().put(application, applicationPackage,"1.0." + buildNumber + "-commit1"); - controllerTester.jobCompletion(DeploymentJobs.JobType.component) + controllerTester.jobCompletion(JobType.component) .application(application) .projectId(projectId) .buildNumber(buildNumber) @@ -1078,7 +1079,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request(testPath, DELETE) .screwdriverIdentity(SCREWDRIVER_ID), "Deactivated " + testPath.replaceFirst("/application/v4/", "")); - controllerTester.jobCompletion(DeploymentJobs.JobType.systemTest) + controllerTester.jobCompletion(JobType.systemTest) .application(application) .projectId(projectId) .submit(); @@ -1093,7 +1094,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request(stagingPath, DELETE) .screwdriverIdentity(SCREWDRIVER_ID), "Deactivated " + stagingPath.replaceFirst("/application/v4/", "")); - controllerTester.jobCompletion(DeploymentJobs.JobType.stagingTest) + controllerTester.jobCompletion(JobType.stagingTest) .application(application) .projectId(projectId) .submit(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index fa93c0f1df1..61f915f3c1e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -8,9 +8,9 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -102,20 +102,20 @@ public class DeploymentApiTest extends ControllerContainerTest { private void deployCompletely(Application application, ApplicationPackage applicationPackage, long projectId, boolean success) { - tester.jobCompletion(DeploymentJobs.JobType.component) + tester.jobCompletion(JobType.component) .application(application) .projectId(projectId) .uploadArtifact(applicationPackage) .submit(); tester.deploy(application, applicationPackage, ZoneId.from(Environment.test, RegionName.from("us-east-1")), projectId); - tester.jobCompletion(DeploymentJobs.JobType.systemTest) + tester.jobCompletion(JobType.systemTest) .application(application) .projectId(projectId) .submit(); tester.deploy(application, applicationPackage, ZoneId.from(Environment.staging, RegionName.from("us-east-3")), projectId); - tester.jobCompletion(DeploymentJobs.JobType.stagingTest) + tester.jobCompletion(JobType.stagingTest) .application(application) .projectId(projectId) .success(success) @@ -123,7 +123,7 @@ public class DeploymentApiTest extends ControllerContainerTest { if (success) { tester.deploy(application, applicationPackage, ZoneId.from(Environment.prod, RegionName.from("corp-us-east-1")), projectId); - tester.jobCompletion(DeploymentJobs.JobType.productionCorpUsEast1) + tester.jobCompletion(JobType.productionCorpUsEast1) .application(application) .projectId(projectId) .submit(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java index 99274fd9f44..6cd4464dce4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.screwdriver; import com.yahoo.application.container.handler.Request; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; @@ -18,7 +18,7 @@ import java.util.OptionalLong; /** * @author bratseth - * @author jvenstad + * @author jonmv */ public class ScrewdriverApiTest extends ControllerContainerTest { @@ -62,7 +62,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { new byte[0], Request.Method.POST, () -> "user"), 200, "{\"message\":\"Triggered component for tenant1.application1\"}"); tester.controller().applications().deploymentTrigger().notifyOfCompletion(new JobReport(app.id(), - DeploymentJobs.JobType.component, + JobType.component, 1, 42, Optional.of(new SourceRevision("repo", "branch", "commit")), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 57676ffc3f5..09216eec3c7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -24,10 +24,10 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/dist/getversion.pl b/dist/getversion.pl index 77271269aa3..80d35d58f43 100755 --- a/dist/getversion.pl +++ b/dist/getversion.pl @@ -5,20 +5,14 @@ use POSIX qw(strftime); $srcdir = "."; -my $generatejava = 0; -my $printdefines = 0; my $printmap = 0; -my $simple = 0; -my $target = ""; -my $pkgname = ""; -my $tagtype = ""; while ($opt = shift) { if ($opt =~ m/^-/) { if ($opt eq "-M") { $printmap = 1; } elsif ($opt eq "-T") { - $tagtype = shift; + die "option '-T' is removed"; } else { print STDERR "ERROR: unknown option '$opt' for getversion\n"; print "error\n"; @@ -29,32 +23,21 @@ while ($opt = shift) { } } -if (!defined($srcdir)) { - die "srcdir must be set"; -} - # Read current major-minor release sub read_head_version() { my $file = "$srcdir/VERSION"; if (! -f $file) { - die "Unable to locate version file"; + die "Unable to locate version file in $srcdir"; } open(my $fd, "< $file") || die "Unable to open VERSION: $!"; my $version = <$fd>; chomp($version); close($fd); - return $version; } -if ( ! -d $srcdir ) { - print STDERR "ERROR: bad directory '$srcdir' for getversion\n"; - print "error\n"; - exit 1; -} - -# assume HEAD if all else fails +# assume HEAD my $mainver = read_head_version(); # date adding logic @@ -63,11 +46,8 @@ my $mainver = read_head_version(); # vbuild/mbuild also has some logic for this: $dateadd = $ENV{"VBUILD_VERSION_DATE"}; -$buildtime = $ENV{"CVSBUILDTIME"}; -if ($buildtime && $buildtime =~ m/^(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)$/ ) { - $dateadd = ".$1$2$3.$4$5$6"; -} elsif ($dateadd) { +if ($dateadd) { 1; } else { $dateadd = (strftime ".%Y%m%d.%H%M%S", gmtime); @@ -77,18 +57,14 @@ $tag = "HEAD"; if (defined $ENV{FACTORY_VESPA_VERSION}) { $version = $ENV{FACTORY_VESPA_VERSION}; -} elsif ($tagtype eq 'tag') { - # only for exact tag, do not add date: - $version = $mainver; } else { $version = $mainver . $dateadd; } -if ($printdefines || $generatejava || $printmap) { +if ($printmap) { # other useful information chomp($ostype = `uname -s`); - chomp($osver = `uname -r`); chomp($osarch = `uname -m`); diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 05ec2ce99d0..2433bfecdab 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -163,8 +163,7 @@ DocumentUpdate::SP DocumentSelectParserTest::createUpdate( const std::string& hstr) { const DocumentType* type = _repo->getDocumentType(doctype); - DocumentUpdate::SP doc( - new DocumentUpdate(*type, DocumentId(id))); + DocumentUpdate::SP doc(new DocumentUpdate(*_repo, *type, DocumentId(id))); doc->addUpdate(FieldUpdate(doc->getType().getField("headerval")) .addUpdate(AssignValueUpdate(IntFieldValue(hint)))); doc->addUpdate(FieldUpdate(doc->getType().getField("hstringval")) diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 9ba17d95264..eea5dd4decf 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -119,36 +119,28 @@ ByteBuffer::UP serialize42(const DocumentUpdate & update) return retVal; } -ByteBuffer::UP serialize(const ValueUpdate & update) +nbostream serialize(const ValueUpdate & update) { nbostream stream; VespaDocumentSerializer serializer(stream); serializer.write(update); - ByteBuffer::UP retVal(new ByteBuffer(stream.size())); - retVal->putBytes(stream.peek(), stream.size()); - return retVal; + return stream; } -ByteBuffer::UP serialize(const FieldUpdate & update) +nbostream serialize(const FieldUpdate & update) { nbostream stream; VespaDocumentSerializer serializer(stream); serializer.write(update); - ByteBuffer::UP retVal(new ByteBuffer(stream.size())); - retVal->putBytes(stream.peek(), stream.size()); - return retVal; + return stream; } template<typename UpdateType> void testValueUpdate(const UpdateType& update, const DataType &type) { try{ DocumentTypeRepo repo; - ByteBuffer::UP buf = serialize(update); - buf->flip(); - typename UpdateType::UP copy(dynamic_cast<UpdateType*>( - ValueUpdate::createInstance(repo, type, *buf, - Document::getNewestSerializationVersion()) - .release())); + nbostream stream = serialize(update); + typename UpdateType::UP copy(dynamic_cast<UpdateType*>(ValueUpdate::createInstance(repo, type, stream).release())); CPPUNIT_ASSERT_EQUAL(update, *copy); } catch (std::exception& e) { std::cerr << "Failed while processing update " << update << "\n"; @@ -189,18 +181,16 @@ DocumentUpdateTest::testSimpleUsage() { FieldUpdate fieldUpdate(docType->getField("intf")); fieldUpdate.addUpdate(AssignValueUpdate(IntFieldValue(1))); - ByteBuffer::UP fieldBuf = serialize(fieldUpdate); - fieldBuf->flip(); - FieldUpdate fieldUpdateCopy(repo, *docType, *fieldBuf, - Document::getNewestSerializationVersion()); + nbostream stream = serialize(fieldUpdate); + FieldUpdate fieldUpdateCopy(repo, *docType, stream); CPPUNIT_ASSERT_EQUAL(fieldUpdate, fieldUpdateCopy); // Test that a document update can be serialized - DocumentUpdate docUpdate(*docType, DocumentId("doc::testdoc")); + DocumentUpdate docUpdate(repo, *docType, DocumentId("doc::testdoc")); docUpdate.addUpdate(fieldUpdateCopy); - ByteBuffer::UP docBuf = serialize42(docUpdate); + ByteBuffer::UP docBuf = serializeHEAD(docUpdate); docBuf->flip(); - DocumentUpdate::UP docUpdateCopy(DocumentUpdate::create42(repo, *docBuf)); + auto docUpdateCopy(DocumentUpdate::createHEAD(repo, nbostream(docBuf->getBufferAtPos(), docBuf->getRemaining()))); // Create a test document Document doc(*docType, DocumentId("doc::testdoc")); @@ -214,62 +204,53 @@ DocumentUpdateTest::testSimpleUsage() { // Verify that we can apply simple updates to it { Document updated(doc); - DocumentUpdate upd(*docType, DocumentId("doc::testdoc")); - upd.addUpdate(FieldUpdate(docType->getField("intf")) - .addUpdate(ClearValueUpdate())); + DocumentUpdate upd(repo, *docType, DocumentId("doc::testdoc")); + upd.addUpdate(FieldUpdate(docType->getField("intf")).addUpdate(ClearValueUpdate())); upd.applyTo(updated); CPPUNIT_ASSERT(doc != updated); CPPUNIT_ASSERT(! updated.getValue("intf")); } { Document updated(doc); - DocumentUpdate upd(*docType, DocumentId("doc::testdoc")); - upd.addUpdate(FieldUpdate(docType->getField("intf")) - .addUpdate(AssignValueUpdate(IntFieldValue(15)))); + DocumentUpdate upd(repo, *docType, DocumentId("doc::testdoc")); + upd.addUpdate(FieldUpdate(docType->getField("intf")).addUpdate(AssignValueUpdate(IntFieldValue(15)))); upd.applyTo(updated); CPPUNIT_ASSERT(doc != updated); CPPUNIT_ASSERT_EQUAL(15, updated.getValue("intf")->getAsInt()); } { Document updated(doc); - DocumentUpdate upd(*docType, DocumentId("doc::testdoc")); - upd.addUpdate(FieldUpdate(docType->getField("intf")) - .addUpdate(ArithmeticValueUpdate( - ArithmeticValueUpdate::Add, 15))); + DocumentUpdate upd(repo, *docType, DocumentId("doc::testdoc")); + upd.addUpdate(FieldUpdate(docType->getField("intf")).addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 15))); upd.applyTo(updated); CPPUNIT_ASSERT(doc != updated); CPPUNIT_ASSERT_EQUAL(20, updated.getValue("intf")->getAsInt()); } { Document updated(doc); - DocumentUpdate upd(*docType, DocumentId("doc::testdoc")); - upd.addUpdate(FieldUpdate(docType->getField("intarr")) - .addUpdate(AddValueUpdate(IntFieldValue(4)))); + DocumentUpdate upd(repo, *docType, DocumentId("doc::testdoc")); + upd.addUpdate(FieldUpdate(docType->getField("intarr")).addUpdate(AddValueUpdate(IntFieldValue(4)))); upd.applyTo(updated); CPPUNIT_ASSERT(doc != updated); - std::unique_ptr<ArrayFieldValue> val(dynamic_cast<ArrayFieldValue*>( - updated.getValue("intarr").release())); + std::unique_ptr<ArrayFieldValue> val(dynamic_cast<ArrayFieldValue*>(updated.getValue("intarr").release())); CPPUNIT_ASSERT_EQUAL(size_t(3), val->size()); CPPUNIT_ASSERT_EQUAL(4, (*val)[2].getAsInt()); } { Document updated(doc); - DocumentUpdate upd(*docType, DocumentId("doc::testdoc")); - upd.addUpdate(FieldUpdate(docType->getField("intarr")) - .addUpdate(RemoveValueUpdate(IntFieldValue(3)))); + DocumentUpdate upd(repo, *docType, DocumentId("doc::testdoc")); + upd.addUpdate(FieldUpdate(docType->getField("intarr")).addUpdate(RemoveValueUpdate(IntFieldValue(3)))); upd.applyTo(updated); CPPUNIT_ASSERT(doc != updated); - std::unique_ptr<ArrayFieldValue> val(dynamic_cast<ArrayFieldValue*>( - updated.getValue("intarr").release())); + std::unique_ptr<ArrayFieldValue> val(dynamic_cast<ArrayFieldValue*>(updated.getValue("intarr").release())); CPPUNIT_ASSERT_EQUAL(size_t(1), val->size()); CPPUNIT_ASSERT_EQUAL(7, (*val)[0].getAsInt()); } { Document updated(doc); - DocumentUpdate upd(*docType, DocumentId("doc::testdoc")); + DocumentUpdate upd(repo, *docType, DocumentId("doc::testdoc")); upd.addUpdate(FieldUpdate(docType->getField("bytef")) - .addUpdate(ArithmeticValueUpdate( - ArithmeticValueUpdate::Add, 15))); + .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 15))); upd.applyTo(updated); CPPUNIT_ASSERT(doc != updated); CPPUNIT_ASSERT_EQUAL(15, (int) updated.getValue("bytef")->getAsByte()); @@ -286,9 +267,8 @@ DocumentUpdateTest::testClearField() CPPUNIT_ASSERT_EQUAL(4, doc->getValue("headerval")->getAsInt()); // Apply an update. - DocumentUpdate(*doc->getDataType(), doc->getId()) - .addUpdate(FieldUpdate(doc->getField("headerval")) - .addUpdate(AssignValueUpdate())) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) + .addUpdate(FieldUpdate(doc->getField("headerval")).addUpdate(AssignValueUpdate())) .applyTo(*doc); CPPUNIT_ASSERT(!doc->getValue("headerval")); } @@ -303,9 +283,8 @@ DocumentUpdateTest::testUpdateApplySingleValue() CPPUNIT_ASSERT_EQUAL(4, doc->getValue("headerval")->getAsInt()); // Apply an update. - DocumentUpdate(*doc->getDataType(), doc->getId()) - .addUpdate(FieldUpdate(doc->getField("headerval")) - .addUpdate(AssignValueUpdate(IntFieldValue(9)))) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) + .addUpdate(FieldUpdate(doc->getField("headerval")).addUpdate(AssignValueUpdate(IntFieldValue(9)))) .applyTo(*doc); CPPUNIT_ASSERT_EQUAL(9, doc->getValue("headerval")->getAsInt()); } @@ -316,28 +295,23 @@ DocumentUpdateTest::testUpdateArray() // Create a document. TestDocMan docMan; Document::UP doc(docMan.createDocument()); - CPPUNIT_ASSERT_EQUAL((document::FieldValue*)NULL, - doc->getValue(doc->getField("tags")).get()); + CPPUNIT_ASSERT_EQUAL((document::FieldValue*)NULL, doc->getValue(doc->getField("tags")).get()); // Assign array field. ArrayFieldValue myarray(doc->getType().getField("tags").getDataType()); myarray.add(StringFieldValue("foo")); myarray.add(StringFieldValue("bar")); - DocumentUpdate(*doc->getDataType(), doc->getId()) - .addUpdate(FieldUpdate(doc->getField("tags")) - .addUpdate(AssignValueUpdate(myarray))) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) + .addUpdate(FieldUpdate(doc->getField("tags")).addUpdate(AssignValueUpdate(myarray))) .applyTo(*doc); - std::unique_ptr<ArrayFieldValue> - fval1(doc->getAs<ArrayFieldValue>(doc->getField("tags"))); + auto fval1(doc->getAs<ArrayFieldValue>(doc->getField("tags"))); CPPUNIT_ASSERT_EQUAL((size_t) 2, fval1->size()); - CPPUNIT_ASSERT_EQUAL(std::string("foo"), - std::string((*fval1)[0].getAsString())); - CPPUNIT_ASSERT_EQUAL(std::string("bar"), - std::string((*fval1)[1].getAsString())); + CPPUNIT_ASSERT_EQUAL(std::string("foo"), std::string((*fval1)[0].getAsString())); + CPPUNIT_ASSERT_EQUAL(std::string("bar"), std::string((*fval1)[1].getAsString())); // Append array field - DocumentUpdate(*doc->getDataType(), doc->getId()) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) .addUpdate(FieldUpdate(doc->getField("tags")) .addUpdate(AddValueUpdate(StringFieldValue("another"))) .addUpdate(AddValueUpdate(StringFieldValue("tag")))) @@ -345,21 +319,16 @@ DocumentUpdateTest::testUpdateArray() std::unique_ptr<ArrayFieldValue> fval2(doc->getAs<ArrayFieldValue>(doc->getField("tags"))); CPPUNIT_ASSERT_EQUAL((size_t) 4, fval2->size()); - CPPUNIT_ASSERT_EQUAL(std::string("foo"), - std::string((*fval2)[0].getAsString())); - CPPUNIT_ASSERT_EQUAL(std::string("bar"), - std::string((*fval2)[1].getAsString())); - CPPUNIT_ASSERT_EQUAL(std::string("another"), - std::string((*fval2)[2].getAsString())); - CPPUNIT_ASSERT_EQUAL(std::string("tag"), - std::string((*fval2)[3].getAsString())); + CPPUNIT_ASSERT_EQUAL(std::string("foo"), std::string((*fval2)[0].getAsString())); + CPPUNIT_ASSERT_EQUAL(std::string("bar"), std::string((*fval2)[1].getAsString())); + CPPUNIT_ASSERT_EQUAL(std::string("another"), std::string((*fval2)[2].getAsString())); + CPPUNIT_ASSERT_EQUAL(std::string("tag"), std::string((*fval2)[3].getAsString())); // Append single value. try { - DocumentUpdate(*doc->getDataType(), doc->getId()) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) .addUpdate(FieldUpdate(doc->getField("tags")) - .addUpdate(AssignValueUpdate( - StringFieldValue("THROW MEH!")))) + .addUpdate(AssignValueUpdate(StringFieldValue("THROW MEH!")))) .applyTo(*doc); CPPUNIT_FAIL("Expected exception when assinging a string value to an " "array field."); @@ -369,25 +338,22 @@ DocumentUpdateTest::testUpdateArray() } // Remove array field. - DocumentUpdate(*doc->getDataType(), doc->getId()) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) .addUpdate(FieldUpdate(doc->getField("tags")) .addUpdate(RemoveValueUpdate(StringFieldValue("foo"))) .addUpdate(RemoveValueUpdate(StringFieldValue("tag")))) .applyTo(*doc); - std::unique_ptr<ArrayFieldValue> - fval3(doc->getAs<ArrayFieldValue>(doc->getField("tags"))); + auto fval3(doc->getAs<ArrayFieldValue>(doc->getField("tags"))); CPPUNIT_ASSERT_EQUAL((size_t) 2, fval3->size()); - CPPUNIT_ASSERT_EQUAL(std::string("bar"), - std::string((*fval3)[0].getAsString())); - CPPUNIT_ASSERT_EQUAL(std::string("another"), - std::string((*fval3)[1].getAsString())); + CPPUNIT_ASSERT_EQUAL(std::string("bar"), std::string((*fval3)[0].getAsString())); + CPPUNIT_ASSERT_EQUAL(std::string("another"), std::string((*fval3)[1].getAsString())); // Remove array from array. ArrayFieldValue myarray2(doc->getType().getField("tags").getDataType()); myarray2.add(StringFieldValue("foo")); myarray2.add(StringFieldValue("bar")); try { - DocumentUpdate(*doc->getDataType(), doc->getId()) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) .addUpdate(FieldUpdate(doc->getField("tags")) .addUpdate(RemoveValueUpdate(myarray2))) .applyTo(*doc); @@ -412,12 +378,10 @@ DocumentUpdateTest::testUpdateWeightedSet() WeightedSetFieldValue wset(field.getDataType()); wset.add(StringFieldValue("foo"), 3); wset.add(StringFieldValue("bar"), 14); - DocumentUpdate(*doc->getDataType(), doc->getId()) - .addUpdate(FieldUpdate(field) - .addUpdate(AssignValueUpdate(wset))) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) + .addUpdate(FieldUpdate(field).addUpdate(AssignValueUpdate(wset))) .applyTo(*doc); - std::unique_ptr<WeightedSetFieldValue> - fval1(doc->getAs<WeightedSetFieldValue>(field)); + auto fval1(doc->getAs<WeightedSetFieldValue>(field)); CPPUNIT_ASSERT_EQUAL((size_t) 2, fval1->size()); CPPUNIT_ASSERT(fval1->contains(StringFieldValue("foo"))); CPPUNIT_ASSERT(fval1->find(StringFieldValue("foo")) != fval1->end()); @@ -430,12 +394,11 @@ DocumentUpdateTest::testUpdateWeightedSet() WeightedSetFieldValue wset2(field.getDataType()); wset2.add(StringFieldValue("foo"), 16); wset2.add(StringFieldValue("bar"), 24); - DocumentUpdate(*doc->getDataType(), doc->getId()) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) .addUpdate(FieldUpdate(field) .addUpdate(AssignValueUpdate(wset2))) .applyTo(*doc); - std::unique_ptr<WeightedSetFieldValue> - fval2(doc->getAs<WeightedSetFieldValue>(field)); + auto fval2(doc->getAs<WeightedSetFieldValue>(field)); CPPUNIT_ASSERT_EQUAL((size_t) 2, fval2->size()); CPPUNIT_ASSERT(fval2->contains(StringFieldValue("foo"))); CPPUNIT_ASSERT(fval2->find(StringFieldValue("foo")) != fval1->end()); @@ -445,12 +408,10 @@ DocumentUpdateTest::testUpdateWeightedSet() CPPUNIT_ASSERT_EQUAL(24, fval2->get(StringFieldValue("bar"), 0)); // Append weighted field - DocumentUpdate(*doc->getDataType(), doc->getId()) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) .addUpdate(FieldUpdate(field) - .addUpdate(AddValueUpdate(StringFieldValue("foo")) - .setWeight(3)) - .addUpdate(AddValueUpdate(StringFieldValue("too")) - .setWeight(14))) + .addUpdate(AddValueUpdate(StringFieldValue("foo")).setWeight(3)) + .addUpdate(AddValueUpdate(StringFieldValue("too")).setWeight(14))) .applyTo(*doc); std::unique_ptr<WeightedSetFieldValue> fval3(doc->getAs<WeightedSetFieldValue>(field)); @@ -463,13 +424,12 @@ DocumentUpdateTest::testUpdateWeightedSet() CPPUNIT_ASSERT_EQUAL(14, fval3->get(StringFieldValue("too"), 0)); // Remove weighted field - DocumentUpdate(*doc->getDataType(), doc->getId()) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) .addUpdate(FieldUpdate(field) .addUpdate(RemoveValueUpdate(StringFieldValue("foo"))) .addUpdate(RemoveValueUpdate(StringFieldValue("too")))) .applyTo(*doc); - std::unique_ptr<WeightedSetFieldValue> - fval4(doc->getAs<WeightedSetFieldValue>(field)); + auto fval4(doc->getAs<WeightedSetFieldValue>(field)); CPPUNIT_ASSERT_EQUAL((size_t) 1, fval4->size()); CPPUNIT_ASSERT(!fval4->contains(StringFieldValue("foo"))); CPPUNIT_ASSERT(fval4->contains(StringFieldValue("bar"))); @@ -500,21 +460,18 @@ struct WeightedSetAutoCreateFixture // and remove-if-zero attributes set. Attempting to explicitly create // a field matching those characteristics will in fact fail with a // redefinition error. - builder.document(42, "test", - Struct("test.header") - .addField("strwset", DataType::T_TAG), - Struct("test.body")); + builder.document(42, "test", Struct("test.header").addField("strwset", DataType::T_TAG), Struct("test.body")); return builder.config(); } }; -WeightedSetAutoCreateFixture::~WeightedSetAutoCreateFixture() {} +WeightedSetAutoCreateFixture::~WeightedSetAutoCreateFixture() = default; WeightedSetAutoCreateFixture::WeightedSetAutoCreateFixture() : repo(makeConfig()), docType(repo.getDocumentType("test")), doc(*docType, DocumentId("doc::testdoc")), field(docType->getField("strwset")), - update(*docType, DocumentId("doc::testdoc")) + update(repo, *docType, DocumentId("doc::testdoc")) { update.addUpdate(FieldUpdate(field) .addUpdate(MapValueUpdate(StringFieldValue("foo"), @@ -585,7 +542,8 @@ void DocumentUpdateTest::testReadSerializedFile() } close(fd); - DocumentUpdate::UP updp(DocumentUpdate::create42(repo, buf)); + nbostream is(buf.getBufferAtPos(), buf.getRemaining()); + DocumentUpdate::UP updp(DocumentUpdate::create42(repo, is)); DocumentUpdate& upd(*updp); const DocumentType *type = repo.getDocumentType("serializetest"); @@ -647,7 +605,7 @@ void DocumentUpdateTest::testGenerateSerializedFile() DocumentTypeRepo repo(readDocumenttypesConfig(file_name)); const DocumentType *type(repo.getDocumentType("serializetest")); - DocumentUpdate upd(*type, DocumentId(DocIdString("update", "test"))); + DocumentUpdate upd(repo, *type, DocumentId(DocIdString("update", "test"))); upd.addUpdate(FieldUpdate(type->getField("intfield")) .addUpdate(AssignValueUpdate(IntFieldValue(4)))); upd.addUpdate(FieldUpdate(type->getField("floatfield")) @@ -678,11 +636,10 @@ void DocumentUpdateTest::testSetBadFieldTypes() // Create a test document TestDocMan docMan; Document::UP doc(docMan.createDocument()); - CPPUNIT_ASSERT_EQUAL((document::FieldValue*)NULL, - doc->getValue(doc->getField("headerval")).get()); + CPPUNIT_ASSERT_EQUAL((document::FieldValue*)NULL, doc->getValue(doc->getField("headerval")).get()); // Assign a float value to an int field. - DocumentUpdate update(*doc->getDataType(), doc->getId()); + DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); try { update.addUpdate(FieldUpdate(doc->getField("headerval")) .addUpdate(AssignValueUpdate(FloatFieldValue(4.00f)))); @@ -701,23 +658,19 @@ void DocumentUpdateTest::testSetBadFieldTypes() void DocumentUpdateTest::testUpdateApplyNoParams() { - // Create a test document TestDocMan docMan; Document::UP doc(docMan.createDocument()); - CPPUNIT_ASSERT_EQUAL((document::FieldValue*)NULL, - doc->getValue(doc->getField("tags")).get()); + CPPUNIT_ASSERT_EQUAL((document::FieldValue*)NULL, doc->getValue(doc->getField("tags")).get()); // Assign array field with no parameters - illegal. - DocumentUpdate update(*doc->getDataType(), doc->getId()); + DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); try { - update.addUpdate(FieldUpdate(doc->getField("tags")) - .addUpdate(AssignValueUpdate())); + update.addUpdate(FieldUpdate(doc->getField("tags")).addUpdate(AssignValueUpdate())); CPPUNIT_FAIL("Expected exception when assign a NULL value."); } catch (std::exception& e) { ; // fprintf(stderr, "Got exception => OK: %s\n", e.what()); } - // Apply update update.applyTo(*doc); // Verify that the field was cleared in the document. @@ -727,20 +680,16 @@ DocumentUpdateTest::testUpdateApplyNoParams() void DocumentUpdateTest::testUpdateApplyNoArrayValues() { - // Create a test document TestDocMan docMan; Document::UP doc(docMan.createDocument()); const Field &field(doc->getType().getField("tags")); - CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, - doc->getValue(field).get()); + CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get()); // Assign array field with no array values = empty array - DocumentUpdate update(*doc->getDataType(), doc->getId()); + DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); update.addUpdate(FieldUpdate(field) - .addUpdate(AssignValueUpdate( - ArrayFieldValue(field.getDataType())))); + .addUpdate(AssignValueUpdate(ArrayFieldValue(field.getDataType())))); - // Apply update update.applyTo(*doc); // Verify that the field was set in the document @@ -759,7 +708,7 @@ DocumentUpdateTest::testUpdateArrayEmptyParamValue() CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get()); // Assign array field with no array values = empty array. - DocumentUpdate update(*doc->getDataType(), doc->getId()); + DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); update.addUpdate(FieldUpdate(field).addUpdate(AssignValueUpdate(ArrayFieldValue(field.getDataType())))); update.applyTo(*doc); @@ -769,7 +718,7 @@ DocumentUpdateTest::testUpdateArrayEmptyParamValue() CPPUNIT_ASSERT_EQUAL((size_t) 0, fval1->size()); // Remove array field. - DocumentUpdate update2(*doc->getDataType(), doc->getId()); + DocumentUpdate update2(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); update2.addUpdate(FieldUpdate(field).addUpdate(ClearValueUpdate())); update2.applyTo(*doc); @@ -788,7 +737,7 @@ DocumentUpdateTest::testUpdateWeightedSetEmptyParamValue() CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get()); // Assign weighted set with no items = empty set. - DocumentUpdate update(*doc->getDataType(), doc->getId()); + DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); update.addUpdate(FieldUpdate(field).addUpdate(AssignValueUpdate(WeightedSetFieldValue(field.getDataType())))); update.applyTo(*doc); @@ -798,7 +747,7 @@ DocumentUpdateTest::testUpdateWeightedSetEmptyParamValue() CPPUNIT_ASSERT_EQUAL((size_t) 0, fval1->size()); // Remove weighted set field. - DocumentUpdate update2(*doc->getDataType(), doc->getId()); + DocumentUpdate update2(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); update2.addUpdate(FieldUpdate(field).addUpdate(ClearValueUpdate())); update2.applyTo(*doc); @@ -817,7 +766,7 @@ DocumentUpdateTest::testUpdateArrayWrongSubtype() CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get()); // Assign int values to string array. - DocumentUpdate update(*doc->getDataType(), doc->getId()); + DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); try { update.addUpdate(FieldUpdate(field) .addUpdate(AddValueUpdate(IntFieldValue(123))) @@ -842,17 +791,14 @@ DocumentUpdateTest::testUpdateWeightedSetWrongSubtype() TestDocMan docMan; Document::UP doc(docMan.createDocument()); const Field &field(doc->getType().getField("stringweightedset")); - CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, - doc->getValue(field).get()); + CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get()); // Assign int values to string array. - DocumentUpdate update(*doc->getDataType(), doc->getId()); + DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); try { update.addUpdate(FieldUpdate(field) - .addUpdate(AddValueUpdate(IntFieldValue(123)) - .setWeight(1000)) - .addUpdate(AddValueUpdate(IntFieldValue(456)) - .setWeight(2000))); + .addUpdate(AddValueUpdate(IntFieldValue(123)).setWeight(1000)) + .addUpdate(AddValueUpdate(IntFieldValue(456)).setWeight(2000))); CPPUNIT_FAIL("Expected exception when adding wrong type."); } catch (std::exception& e) { ; // fprintf(stderr, "Got exception => OK: %s\n", e.what()); @@ -879,79 +825,61 @@ DocumentUpdateTest::testMapValueUpdate() doc->setValue(field1, wsval1); doc->setValue(field2, wsval2); - DocumentUpdate(*doc->getDataType(), doc->getId()) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) .addUpdate(FieldUpdate(field1) - .addUpdate(MapValueUpdate( - StringFieldValue("banana"), - ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 1.0) - ))) + .addUpdate(MapValueUpdate(StringFieldValue("banana"), + ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 1.0)))) .applyTo(*doc); std::unique_ptr<WeightedSetFieldValue> fv1 = doc->getAs<WeightedSetFieldValue>(field1); CPPUNIT_ASSERT(fv1->size() == 0); - DocumentUpdate(*doc->getDataType(), doc->getId()) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) .addUpdate(FieldUpdate(field2) - .addUpdate(MapValueUpdate( - StringFieldValue("banana"), - ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 1.0) - ))) + .addUpdate(MapValueUpdate(StringFieldValue("banana"), + ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 1.0)))) .applyTo(*doc); - std::unique_ptr<WeightedSetFieldValue> fv2 = - doc->getAs<WeightedSetFieldValue>(field2); + auto fv2 = doc->getAs<WeightedSetFieldValue>(field2); CPPUNIT_ASSERT(fv2->size() == 1); CPPUNIT_ASSERT(fv1->find(StringFieldValue("apple")) == fv1->end()); - DocumentUpdate(*doc->getDataType(), doc->getId()) - .addUpdate(FieldUpdate(field1) - .addUpdate(ClearValueUpdate())) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) + .addUpdate(FieldUpdate(field1).addUpdate(ClearValueUpdate())) .applyTo(*doc); - DocumentUpdate(*doc->getDataType(), doc->getId()) - .addUpdate(FieldUpdate(field1) - .addUpdate(AddValueUpdate(StringFieldValue("apple")) - .setWeight(1))) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) + .addUpdate(FieldUpdate(field1).addUpdate(AddValueUpdate(StringFieldValue("apple")).setWeight(1))) .applyTo(*doc); - std::unique_ptr<WeightedSetFieldValue> - fval3(doc->getAs<WeightedSetFieldValue>(field1)); + auto fval3(doc->getAs<WeightedSetFieldValue>(field1)); CPPUNIT_ASSERT(fval3->find(StringFieldValue("apple")) != fval3->end()); CPPUNIT_ASSERT_EQUAL(1, fval3->get(StringFieldValue("apple"))); - DocumentUpdate(*doc->getDataType(), doc->getId()) - .addUpdate(FieldUpdate(field2) - .addUpdate(AddValueUpdate(StringFieldValue("apple")) - .setWeight(1))) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) + .addUpdate(FieldUpdate(field2).addUpdate(AddValueUpdate(StringFieldValue("apple")).setWeight(1))) .applyTo(*doc); - std::unique_ptr<WeightedSetFieldValue> - fval3b(doc->getAs<WeightedSetFieldValue>(field2)); + auto fval3b(doc->getAs<WeightedSetFieldValue>(field2)); CPPUNIT_ASSERT(fval3b->find(StringFieldValue("apple")) != fval3b->end()); CPPUNIT_ASSERT_EQUAL(1, fval3b->get(StringFieldValue("apple"))); - DocumentUpdate(*doc->getDataType(), doc->getId()) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) .addUpdate(FieldUpdate(field1) - .addUpdate(MapValueUpdate( - StringFieldValue("apple"), - ArithmeticValueUpdate(ArithmeticValueUpdate::Sub, 1.0) - ))) + .addUpdate(MapValueUpdate(StringFieldValue("apple"), + ArithmeticValueUpdate(ArithmeticValueUpdate::Sub, 1.0)))) .applyTo(*doc); - std::unique_ptr<WeightedSetFieldValue> fv3 = - doc->getAs<WeightedSetFieldValue>(field1); + auto fv3 = doc->getAs<WeightedSetFieldValue>(field1); CPPUNIT_ASSERT(fv3->find(StringFieldValue("apple")) != fv3->end()); CPPUNIT_ASSERT_EQUAL(0, fv3->get(StringFieldValue("apple"))); - DocumentUpdate(*doc->getDataType(), doc->getId()) + DocumentUpdate(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()) .addUpdate(FieldUpdate(field2) - .addUpdate(MapValueUpdate( - StringFieldValue("apple"), - ArithmeticValueUpdate(ArithmeticValueUpdate::Sub, 1.0) - ))) + .addUpdate(MapValueUpdate(StringFieldValue("apple"), + ArithmeticValueUpdate(ArithmeticValueUpdate::Sub, 1.0)))) .applyTo(*doc); - std::unique_ptr<WeightedSetFieldValue> fv4 = - doc->getAs<WeightedSetFieldValue>(field2); + auto fv4 = doc->getAs<WeightedSetFieldValue>(field2); CPPUNIT_ASSERT(fv4->find(StringFieldValue("apple")) == fv4->end()); } @@ -965,9 +893,8 @@ DocumentUpdateTest::testTensorAssignUpdate() Document updated(*doc); FieldValue::UP new_value(createTensorFieldValue()); testValueUpdate(AssignValueUpdate(*new_value), *DataType::TENSOR); - DocumentUpdate upd(*doc->getDataType(), doc->getId()); - upd.addUpdate(FieldUpdate(upd.getType().getField("tensor")). - addUpdate(AssignValueUpdate(*new_value))); + DocumentUpdate upd(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); + upd.addUpdate(FieldUpdate(upd.getType().getField("tensor")).addUpdate(AssignValueUpdate(*new_value))); upd.applyTo(updated); FieldValue::UP fval(updated.getValue("tensor")); CPPUNIT_ASSERT(fval); @@ -983,9 +910,8 @@ DocumentUpdateTest::testTensorClearUpdate() Document updated(*doc); updated.setValue(updated.getField("tensor"), *createTensorFieldValue()); CPPUNIT_ASSERT(*doc != updated); - DocumentUpdate upd(*doc->getDataType(), doc->getId()); - upd.addUpdate(FieldUpdate(upd.getType().getField("tensor")). - addUpdate(ClearValueUpdate())); + DocumentUpdate upd(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); + upd.addUpdate(FieldUpdate(upd.getType().getField("tensor")).addUpdate(ClearValueUpdate())); upd.applyTo(updated); CPPUNIT_ASSERT(!updated.getValue("tensor")); CPPUNIT_ASSERT(*doc == updated); @@ -1036,12 +962,11 @@ struct CreateIfNonExistentFixture CreateIfNonExistentFixture(); }; -CreateIfNonExistentFixture::~CreateIfNonExistentFixture() {} +CreateIfNonExistentFixture::~CreateIfNonExistentFixture() = default; CreateIfNonExistentFixture::CreateIfNonExistentFixture() : docMan(), document(docMan.createDocument()), - update(new DocumentUpdate(*document->getDataType(), - document->getId())) + update(new DocumentUpdate(docMan.getTypeRepo(), *document->getDataType(), document->getId())) { update->addUpdate(FieldUpdate(document->getField("headerval")) .addUpdate(AssignValueUpdate(IntFieldValue(1)))); @@ -1069,7 +994,8 @@ DocumentUpdateTest::testThatCreateIfNonExistentFlagIsSerializedAndDeserialized() ByteBuffer::UP buf(serialize42(*f.update)); buf->flip(); - DocumentUpdate::UP deserialized = DocumentUpdate::create42(f.docMan.getTypeRepo(), *buf); + nbostream is(buf->getBufferAtPos(), buf->getRemaining()); + auto deserialized = DocumentUpdate::create42(f.docMan.getTypeRepo(), is); CPPUNIT_ASSERT_EQUAL(*f.update, *deserialized); CPPUNIT_ASSERT(deserialized->getCreateIfNonExistent()); } @@ -1089,7 +1015,7 @@ ArrayUpdateFixture::ArrayUpdateFixture() doc(doc_man.createDocument()), array_field(doc->getType().getField("tags")) // of type array<string> { - update = std::make_unique<DocumentUpdate>(*doc->getDataType(), doc->getId()); + update = std::make_unique<DocumentUpdate>(doc_man.getTypeRepo(), *doc->getDataType(), doc->getId()); update->addUpdate(FieldUpdate(array_field) .addUpdate(MapValueUpdate(IntFieldValue(1), AssignValueUpdate(StringFieldValue("bar"))))); diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index b240f322e4b..80a63a86b92 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -371,7 +371,7 @@ FieldPathUpdateTestCase::testRemoveField() doc->setValue("strfoo", StringFieldValue("cocacola")); CPPUNIT_ASSERT_EQUAL(vespalib::string("cocacola"), doc->getValue("strfoo")->getAsString()); //doc->print(std::cerr, true, ""); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP(new RemoveFieldPathUpdate("strfoo"))); docUp.applyTo(*doc); CPPUNIT_ASSERT(doc->hasValue("strfoo") == false); @@ -392,7 +392,7 @@ FieldPathUpdateTestCase::testApplyRemoveMultiList() } CPPUNIT_ASSERT(doc->hasValue("strarray")); //doc->print(std::cerr, true, ""); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP( new RemoveFieldPathUpdate("strarray[$x]", "foobar.strarray[$x] == \"remove val 1\""))); docUp.applyTo(*doc); @@ -417,7 +417,7 @@ FieldPathUpdateTestCase::testApplyRemoveEntireListField() doc->setValue("strarray", strArray); } //doc->print(std::cerr, true, ""); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP(new RemoveFieldPathUpdate("strarray", ""))); docUp.applyTo(*doc); CPPUNIT_ASSERT(!doc->hasValue("strarray")); @@ -436,7 +436,7 @@ FieldPathUpdateTestCase::testApplyRemoveMultiWset() } CPPUNIT_ASSERT(doc->hasValue("strwset")); //doc->print(std::cerr, true, ""); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP(new RemoveFieldPathUpdate("strwset{remove val 1}"))); docUp.applyTo(*doc); { @@ -452,14 +452,14 @@ FieldPathUpdateTestCase::testApplyAssignSingle() Document::UP doc(new Document(_foobar_type, DocumentId("doc:drekka:karsk"))); CPPUNIT_ASSERT(doc->hasValue("strfoo") == false); // Test assignment of non-existing - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP( new AssignFieldPathUpdate(*doc->getDataType(), "strfoo", std::string(), StringFieldValue("himert")))); docUp.applyTo(*doc); CPPUNIT_ASSERT(doc->hasValue("strfoo")); CPPUNIT_ASSERT_EQUAL(vespalib::string("himert"), doc->getValue("strfoo")->getAsString()); // Test overwriting existing - DocumentUpdate docUp2(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp2(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp2.addFieldPathUpdate(FieldPathUpdate::CP( new AssignFieldPathUpdate(*doc->getDataType(), "strfoo", std::string(), StringFieldValue("wunderbaum")))); docUp2.applyTo(*doc); @@ -472,7 +472,7 @@ FieldPathUpdateTestCase::testApplyAssignMath() Document::UP doc(new Document(_foobar_type, DocumentId("doc:bat:man"))); doc->setValue("num", IntFieldValue(34)); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP(new AssignFieldPathUpdate("num", "", "($value * 2) / $value"))); docUp.applyTo(*doc); CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(IntFieldValue(2)), *doc->getValue("num")); @@ -484,7 +484,7 @@ FieldPathUpdateTestCase::testApplyAssignMathByteToZero() Document::UP doc(new Document(_foobar_type, DocumentId("doc:bat:man"))); doc->setValue("byteval", ByteFieldValue(3)); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP(new AssignFieldPathUpdate("byteval", "", "$value - 3"))); docUp.applyTo(*doc); CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(ByteFieldValue(0)), *doc->getValue("byteval")); @@ -497,7 +497,7 @@ FieldPathUpdateTestCase::testApplyAssignMathNotModifiedOnUnderflow() Document::UP doc(new Document(_foobar_type, DocumentId("doc:bat:man"))); doc->setValue("byteval", ByteFieldValue(low_value)); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP(new AssignFieldPathUpdate("byteval", "", "$value - 4"))); docUp.applyTo(*doc); // Over/underflow will happen. You must have control of your data types. @@ -510,7 +510,7 @@ FieldPathUpdateTestCase::testApplyAssignMathNotModifiedOnOverflow() Document::UP doc(new Document(_foobar_type, DocumentId("doc:bat:man"))); doc->setValue("byteval", ByteFieldValue(127)); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP(new AssignFieldPathUpdate("byteval", "", "$value + 200"))); docUp.applyTo(*doc); // Over/underflow will happen. You must have control of your data types. @@ -524,7 +524,7 @@ FieldPathUpdateTestCase::testApplyAssignMathDivZero() CPPUNIT_ASSERT(doc->hasValue("num") == false); doc->setValue("num", IntFieldValue(10)); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP(new AssignFieldPathUpdate("num", "", "$value / ($value - 10)"))); docUp.applyTo(*doc); CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(IntFieldValue(10)), *doc->getValue("num")); @@ -538,7 +538,7 @@ FieldPathUpdateTestCase::testApplyAssignFieldNotExistingInExpression() CPPUNIT_ASSERT(doc->hasValue("num") == false); doc->setValue("num", IntFieldValue(10)); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP(new AssignFieldPathUpdate("num", "", "foobar.num2 + $value"))); docUp.applyTo(*doc); CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(IntFieldValue(10)), *doc->getValue("num")); @@ -550,7 +550,7 @@ FieldPathUpdateTestCase::testApplyAssignFieldNotExistingInPath() Document::UP doc(new Document(_foobar_type, DocumentId("doc:bat:man"))); doc->setRepo(*_repo); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); try { docUp.addFieldPathUpdate(FieldPathUpdate::CP(new AssignFieldPathUpdate("nosuchnum", "", "foobar.num + $value"))); docUp.applyTo(*doc); @@ -565,7 +565,7 @@ FieldPathUpdateTestCase::testApplyAssignTargetNotExisting() Document::UP doc(new Document(_foobar_type, DocumentId("doc:bat:man"))); CPPUNIT_ASSERT(doc->hasValue("num") == false); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP(new AssignFieldPathUpdate("num", "", "$value + 5"))); docUp.applyTo(*doc); CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(IntFieldValue(5)), *doc->getValue("num")); @@ -582,7 +582,7 @@ FieldPathUpdateTestCase::testAssignSimpleMapValueWithVariable() mfv.put(StringFieldValue("baz"), StringFieldValue("bananas")); doc->setValue("strmap", mfv); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); // Select on value, not key docUp.addFieldPathUpdate(FieldPathUpdate::CP( new AssignFieldPathUpdate(*doc->getDataType(), @@ -608,7 +608,7 @@ FieldPathUpdateTestCase::testApplyAssignMathRemoveIfZero() doc->setValue("num", IntFieldValue(34)); CPPUNIT_ASSERT(doc->hasValue("num") == true); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); FieldPathUpdate::CP up1(new AssignFieldPathUpdate("num", "", "($value * 2) / $value - 2")); static_cast<AssignFieldPathUpdate&>(*up1).setRemoveIfZero(true); docUp.addFieldPathUpdate(up1); @@ -635,7 +635,7 @@ FieldPathUpdateTestCase::testApplyAssignMultiList() updateArray.add(StringFieldValue("assigned val 0")); updateArray.add(StringFieldValue("assigned val 1")); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP( new AssignFieldPathUpdate(*doc->getDataType(), "strarray", std::string(), updateArray))); docUp.applyTo(*doc); @@ -667,7 +667,7 @@ FieldPathUpdateTestCase::testApplyAssignMultiWset() assignWset.add(StringFieldValue("assigned val 0"), 5); assignWset.add(StringFieldValue("assigned val 1"), 10); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP( new AssignFieldPathUpdate(*doc->getDataType(), "strwset", std::string(), assignWset))); //doc->print(std::cerr, true, ""); @@ -696,7 +696,7 @@ FieldPathUpdateTestCase::testAssignWsetRemoveIfZero() } { - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); IntFieldValue zeroWeight(0); FieldPathUpdate::CP assignUpdate( new AssignFieldPathUpdate(*doc->getDataType(), "strwset{you say goodbye}", std::string(), zeroWeight)); @@ -724,7 +724,7 @@ FieldPathUpdateTestCase::testApplyAddMultiList() adds.add(StringFieldValue("a festivus for the rest of us")); adds.add(StringFieldValue("george is getting upset!")); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP( new AddFieldPathUpdate(*doc->getDataType(), "strarray", std::string(), adds))); //doc->print(std::cerr, true, ""); @@ -747,7 +747,7 @@ FieldPathUpdateTestCase::testAddAndAssignList() CPPUNIT_ASSERT(doc->hasValue("strarray")); } - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP( new AssignFieldPathUpdate(*doc->getDataType(), "strarray[1]", std::string(), StringFieldValue("assigned val 1")))); @@ -829,7 +829,7 @@ FieldPathUpdateTestCase::testAssignMap() Keys k; Fixture f(_foobar_type, k); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP( new AssignFieldPathUpdate(*f.doc->getDataType(), "structmap{" + k.key2 + "}", std::string(), f.fv4))); docUp.applyTo(*f.doc); @@ -850,7 +850,7 @@ FieldPathUpdateTestCase::testAssignMapStruct() Keys k; Fixture f(_foobar_type, k); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP( new AssignFieldPathUpdate(*f.doc->getDataType(), "structmap{" + k.key2 + "}.rating", std::string(), IntFieldValue(48)))); @@ -872,7 +872,7 @@ FieldPathUpdateTestCase::testAssignMapStructVariable() Keys k; Fixture f(_foobar_type, k); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP( new AssignFieldPathUpdate(*f.doc->getDataType(), "structmap{$x}.rating", "foobar.structmap{$x}.title == \"farnsworth\"", IntFieldValue(48)))); @@ -899,7 +899,7 @@ FieldPathUpdateTestCase::testAssignMapNoExist() fv1.setValue("title", StringFieldValue("fry")); fv1.setValue("rating", IntFieldValue(30)); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP( new AssignFieldPathUpdate(*doc->getDataType(), "structmap{foo}", std::string(), fv1))); //doc->print(std::cerr, true, ""); @@ -922,7 +922,7 @@ FieldPathUpdateTestCase::testAssignMapNoExistNoCreate() fv1.setValue("title", StringFieldValue("fry")); fv1.setValue("rating", IntFieldValue(30)); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); FieldPathUpdate::CP assignUpdate( new AssignFieldPathUpdate(*doc->getDataType(), "structmap{foo}", std::string(), fv1)); static_cast<AssignFieldPathUpdate&>(*assignUpdate).setCreateMissingPath(false); @@ -944,7 +944,7 @@ FieldPathUpdateTestCase::testQuotedStringKey() const char field_path[] = "structmap{\"here is a \\\"fancy\\\" 'map' :-} key :-{\"}"; Fixture f(_foobar_type, k); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP( new AssignFieldPathUpdate(*f.doc->getDataType(), field_path, std::string(), f.fv4))); docUp.applyTo(*f.doc); @@ -970,8 +970,8 @@ FieldPathUpdateTestCase::testEqualityComparison() fv4.setValue("rating", IntFieldValue(95)); { - DocumentUpdate docUp1(_foobar_type, DocumentId("doc:barbar:foofoo")); - DocumentUpdate docUp2(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp1(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp2(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); CPPUNIT_ASSERT(docUp1 == docUp2); FieldPathUpdate::CP assignUp1(new AssignFieldPathUpdate(*doc->getDataType(), @@ -982,8 +982,8 @@ FieldPathUpdateTestCase::testEqualityComparison() CPPUNIT_ASSERT(docUp1 == docUp2); } { - DocumentUpdate docUp1(_foobar_type, DocumentId("doc:barbar:foofoo")); - DocumentUpdate docUp2(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp1(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp2(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); // where-clause diff FieldPathUpdate::CP assignUp1(new AssignFieldPathUpdate(*doc->getDataType(), "structmap{here be dragons}", std::string(), fv4)); @@ -994,8 +994,8 @@ FieldPathUpdateTestCase::testEqualityComparison() CPPUNIT_ASSERT(docUp1 != docUp2); } { - DocumentUpdate docUp1(_foobar_type, DocumentId("doc:barbar:foofoo")); - DocumentUpdate docUp2(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp1(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp2(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); // fieldpath diff FieldPathUpdate::CP assignUp1(new AssignFieldPathUpdate(*doc->getDataType(), "structmap{here be dragons}", std::string(), fv4)); @@ -1020,7 +1020,7 @@ FieldPathUpdateTestCase::testAffectsDocumentBody() // structmap is body field { - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*doc->getDataType(), "structmap{janitor}", std::string(), fv4)); @@ -1030,7 +1030,7 @@ FieldPathUpdateTestCase::testAffectsDocumentBody() // strfoo is header field { - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*doc->getDataType(), "strfoo", std::string(), StringFieldValue("helloworld"))); static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true); @@ -1045,7 +1045,7 @@ FieldPathUpdateTestCase::testIncompatibleDataTypeFails() Document::UP doc(new Document(_foobar_type, DocumentId("doc:things:stuff"))); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); try { FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*doc->getDataType(), "structmap{foo}", @@ -1066,7 +1066,7 @@ FieldPathUpdateTestCase::testSerializeAssign() val.setValue("title", StringFieldValue("cool frog")); val.setValue("rating", IntFieldValue(100)); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*doc->getDataType(), "structmap{ribbit}", "true", val)); static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true); @@ -1086,7 +1086,7 @@ FieldPathUpdateTestCase::testSerializeAdd() adds.add(StringFieldValue("a festivus for the rest of us")); adds.add(StringFieldValue("george is getting upset!")); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); FieldPathUpdate::CP update1(new AddFieldPathUpdate(*doc->getDataType(), "strarray", std::string(), adds)); docUp.addFieldPathUpdate(update1); @@ -1100,7 +1100,7 @@ FieldPathUpdateTestCase::testSerializeRemove() Document::UP doc(new Document(_foobar_type, DocumentId("doc:weloveto:serializestuff"))); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); FieldPathUpdate::CP update1(new RemoveFieldPathUpdate("structmap{ribbit}", std::string())); docUp.addFieldPathUpdate(update1); @@ -1115,7 +1115,7 @@ FieldPathUpdateTestCase::testSerializeAssignMath() CPPUNIT_ASSERT(doc->hasValue("num") == false); doc->setValue("num", IntFieldValue(34)); - DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); + DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("doc:barbar:foofoo")); docUp.addFieldPathUpdate(FieldPathUpdate::CP(new AssignFieldPathUpdate("num", "", "($value * 2) / $value"))); testSerialize(*_repo, docUp); } @@ -1124,7 +1124,7 @@ DocumentUpdate::UP FieldPathUpdateTestCase::createDocumentUpdateForSerialization(const DocumentTypeRepo& repo) { const DocumentType *docType(repo.getDocumentType("serializetest")); - DocumentUpdate::UP docUp(new DocumentUpdate(*docType, DocumentId("doc:serialization:xlanguage"))); + DocumentUpdate::UP docUp(new DocumentUpdate(repo, *docType, DocumentId("doc:serialization:xlanguage"))); FieldPathUpdate::CP assign(new AssignFieldPathUpdate("intfield", "", "3")); static_cast<AssignFieldPathUpdate&>(*assign).setRemoveIfZero(true); diff --git a/document/src/tests/testxml.cpp b/document/src/tests/testxml.cpp index 4cc1306ff45..09c2ef86279 100644 --- a/document/src/tests/testxml.cpp +++ b/document/src/tests/testxml.cpp @@ -73,7 +73,7 @@ createTestDocumentUpdate(const DocumentTypeRepo& repo) const DocumentType* type(repo.getDocumentType("testdoc")); DocumentId id("doc:crawler/http://www.ntnu.no/"); - DocumentUpdate::UP up(new DocumentUpdate(*type, id)); + DocumentUpdate::UP up(new DocumentUpdate(repo, *type, id)); up->addUpdate(FieldUpdate(type->getField("intattr")) .addUpdate(AssignValueUpdate(IntFieldValue(7)))); up->addUpdate(FieldUpdate(type->getField("stringattr")) diff --git a/document/src/vespa/document/base/forcelink.cpp b/document/src/vespa/document/base/forcelink.cpp index 8f683e40059..bc0097b68f4 100644 --- a/document/src/vespa/document/base/forcelink.cpp +++ b/document/src/vespa/document/base/forcelink.cpp @@ -13,7 +13,7 @@ ForceLink::ForceLink(void) if (time(NULL) == 0) { DocumentType type("foo", 1); Document document(type, DocumentId("doc:ns:bar")); - DocumentUpdate documentUpdate(type, DocumentId("doc:ns:bar")); + DocumentUpdate documentUpdate; MapValueUpdate mapValueUpdate(IntFieldValue(3), ClearValueUpdate()); AddValueUpdate addValueUpdate(IntFieldValue(3)); RemoveValueUpdate removeValueUpdate(IntFieldValue(3)); diff --git a/document/src/vespa/document/fieldvalue/stringfieldvalue.h b/document/src/vespa/document/fieldvalue/stringfieldvalue.h index be32f007db4..14b67838787 100644 --- a/document/src/vespa/document/fieldvalue/stringfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/stringfieldvalue.h @@ -7,15 +7,16 @@ */ #pragma once -#include <vespa/document/fieldvalue/literalfieldvalue.h> +#include "literalfieldvalue.h" #include <vespa/document/annotation/spantree.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/vespalib/util/buffer.h> #include <vespa/fastos/dynamiclibrary.h> namespace document { + class FixedTypeRepo; +class DocumentTypeRepo; class StringFieldValue : public LiteralFieldValue<StringFieldValue, DataType::T_STRING, true> { public: diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index 6bfae246c10..a2f5aeb8f0b 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -51,14 +51,14 @@ using DocumentTypeMap = internal::DocumentTypeMap; namespace { template <typename Container> void DeleteContent(Container &c) { - for (typename Container::iterator it = c.begin(); it != c.end(); ++it) { - delete *it; + for (auto ptr : c) { + delete ptr; } } template <typename Map> void DeleteMapContent(Map &m) { - for (typename Map::iterator it = m.begin(); it != m.end(); ++it) { - delete it->second; + for (auto & entry : m) { + delete entry.second; } } @@ -93,18 +93,14 @@ bool Repo::addDataType(const DataType &type) { return false; // Redefinition of identical type is ok. } throw IllegalArgumentException( - make_string("Redefinition of data type %d, \"%s\". " - "Previously defined as \"%s\".", - type.getId(), type.getName().c_str(), - data_type->getName().c_str())); + make_string("Redefinition of data type %d, \"%s\". Previously defined as \"%s\".", + type.getId(), type.getName().c_str(), data_type->getName().c_str())); } const DataType *& data_type_by_name = _name_map[type.getName()]; if (data_type_by_name) { throw IllegalArgumentException( - make_string("Redefinition of data type \"%s\", with id %d." - " Previously defined with id %d.", - type.getName().c_str(), type.getId(), - data_type_by_name->getId())); + make_string("Redefinition of data type \"%s\", with id %d. Previously defined with id %d.", + type.getName().c_str(), type.getId(), data_type_by_name->getId())); } data_type = &type; data_type_by_name = &type; @@ -158,8 +154,7 @@ public: }; void AnnotationTypeRepo::inherit(const AnnotationTypeRepo &parent) { - _annotation_types.insert(parent._annotation_types.begin(), - parent._annotation_types.end()); + _annotation_types.insert(parent._annotation_types.begin(), parent._annotation_types.end()); } void AnnotationTypeRepo::addAnnotationType(AnnotationType::UP type) { @@ -167,10 +162,8 @@ void AnnotationTypeRepo::addAnnotationType(AnnotationType::UP type) { if (a_type) { if (*type != *a_type) { throw IllegalArgumentException( - make_string("Redefinition of annotation type %d, \"%s\". " - "Previously defined as \"%s\".", - type->getId(), type->getName().c_str(), - a_type->getName().c_str())); + make_string("Redefinition of annotation type %d, \"%s\". Previously defined as \"%s\".", + type->getId(), type->getName().c_str(), a_type->getName().c_str())); } } else { a_type = type.get(); @@ -185,12 +178,9 @@ void AnnotationTypeRepo::setAnnotationDataType(int32_t id, const DataType &d) { annotation_type->setDataType(d); } else if (*(annotation_type->getDataType()) != d) { throw IllegalArgumentException( - make_string("Redefinition of annotation type %d, \"%s\" = '%s'. " - "Previously defined as '%s'.", - annotation_type->getId(), - annotation_type->getName().c_str(), - annotation_type->getDataType()->toString().c_str(), - d.toString().c_str())); + make_string("Redefinition of annotation type %d, \"%s\" = '%s'. Previously defined as '%s'.", + annotation_type->getId(), annotation_type->getName().c_str(), + annotation_type->getDataType()->toString().c_str(), d.toString().c_str())); } } @@ -214,24 +204,22 @@ struct DataTypeRepo { }; namespace { -void addAnnotationType( - const DocumenttypesConfig::Documenttype::Annotationtype &type, - AnnotationTypeRepo &annotations) { +void addAnnotationType(const DocumenttypesConfig::Documenttype::Annotationtype &type, AnnotationTypeRepo &annotations) +{ AnnotationType::UP a(new AnnotationType(type.id, type.name)); annotations.addAnnotationType(std::move(a)); } -void addAnnotationTypes( - const vector<DocumenttypesConfig::Documenttype::Annotationtype> &types, - AnnotationTypeRepo &annotations) { +void addAnnotationTypes(const vector<DocumenttypesConfig::Documenttype::Annotationtype> &types, + AnnotationTypeRepo &annotations) { for (size_t i = 0; i < types.size(); ++i) { addAnnotationType(types[i], annotations); } } -void setAnnotationDataTypes( - const vector<DocumenttypesConfig::Documenttype::Annotationtype> &types, - AnnotationTypeRepo &annotations, const Repo &repo) { +void setAnnotationDataTypes(const vector<DocumenttypesConfig::Documenttype::Annotationtype> &types, + AnnotationTypeRepo &annotations, const Repo &repo) +{ for (size_t i = 0; i < types.size(); ++i) { if (types[i].datatype == -1) { continue; @@ -243,11 +231,10 @@ void setAnnotationDataTypes( typedef DocumenttypesConfig::Documenttype::Datatype Datatype; -void addField(const Datatype::Sstruct::Field &field, const Repo &repo, - StructDataType &struct_type, bool isHeaderField) { +void addField(const Datatype::Sstruct::Field &field, const Repo &repo, StructDataType &struct_type, bool isHeaderField) +{ LOG(spam, "Adding field %s to %s (header: %s)", - field.name.c_str(), struct_type.getName().c_str(), - isHeaderField ? "yes" : "no"); + field.name.c_str(), struct_type.getName().c_str(), isHeaderField ? "yes" : "no"); const DataType &field_type = repo.findOrThrow(field.datatype); struct_type.addField(Field(field.name, field.id, field_type, isHeaderField)); } @@ -270,24 +257,20 @@ void addStruct(int32_t id, const Datatype::Sstruct &s, Repo &repo) { } else if (name.rfind(".header") != std::string::npos) { const DataType *existing = repo.lookup(name); if (existing) { - LOG(spam, "Reusing id %u from body struct since its fields " - "have already been inserted", - existing->getId()); + LOG(spam, "Reusing id %u from body struct since its fields have already been inserted", existing->getId()); id = existing->getId(); } useUglyStructHack = true; } - LOG(debug, "Adding struct type %s (%s) with id %u", - s.name.c_str(), name.c_str(), id); + LOG(debug, "Adding struct type %s (%s) with id %u", s.name.c_str(), name.c_str(), id); StructDataType::UP struct_type_ap; StructDataType *struct_type; const DataType *existing = repo.lookup(name); if (useUglyStructHack && existing) { LOG(spam, "Type %s already existed", name.c_str()); - const StructDataType& cdt = - Identifiable::cast<const StructDataType&>(*existing); + const StructDataType& cdt = Identifiable::cast<const StructDataType&>(*existing); struct_type = const_cast<StructDataType*>(&cdt); } else { const DataType *existing_retry = repo.lookup(id); @@ -306,8 +289,7 @@ void addStruct(int32_t id, const Datatype::Sstruct &s, Repo &repo) { } struct_type->setCompressionConfig( - CompressionConfig(type, s.compression.level, - s.compression.threshold, s.compression.minsize)); + CompressionConfig(type, s.compression.level, s.compression.threshold, s.compression.minsize)); for (size_t i = 0; i < s.field.size(); ++i) { addField(s.field[i], repo, *struct_type, hasSuffix(s.name, ".header")); @@ -321,8 +303,7 @@ void addArray(int32_t id, const Datatype::Array &a, Repo &repo) { void addWset(int32_t id, const Datatype::Wset &w, Repo &repo) { const DataType &key = repo.findOrThrow(w.key.id); - repo.addDataType(DataType::UP(new WeightedSetDataType( - key, w.createifnonexistent, w.removeifzero, id))); + repo.addDataType(DataType::UP(new WeightedSetDataType(key, w.createifnonexistent, w.removeifzero, id))); } void addMap(int32_t id, const Datatype::Map &m, Repo &repo) { @@ -331,18 +312,15 @@ void addMap(int32_t id, const Datatype::Map &m, Repo &repo) { repo.addDataType(DataType::UP(new MapDataType(key, value, id))); } -void addAnnotationRef(int32_t id, const Datatype::Annotationref &a, Repo &r, - const AnnotationTypeRepo &annotations) { +void addAnnotationRef(int32_t id, const Datatype::Annotationref &a, Repo &r, const AnnotationTypeRepo &annotations) { const AnnotationType *type = annotations.lookup(a.annotation.id); if (!type) { - throw IllegalArgumentException( - make_string("Unknown AnnotationType %d", a.annotation.id)); + throw IllegalArgumentException(make_string("Unknown AnnotationType %d", a.annotation.id)); } r.addDataType(DataType::UP(new AnnotationReferenceDataType(*type, id))); } -void addDataType(const Datatype &type, Repo &repo, - const AnnotationTypeRepo &a_repo) { +void addDataType(const Datatype &type, Repo &repo, const AnnotationTypeRepo &a_repo) { switch (type.type) { case Datatype::STRUCT: return addStruct(type.id, type.sstruct, repo); @@ -355,27 +333,24 @@ void addDataType(const Datatype &type, Repo &repo, case Datatype::ANNOTATIONREF: return addAnnotationRef(type.id, type.annotationref, repo, a_repo); default: - throw IllegalArgumentException( - make_string("Unknown datatype type %d for id %d", - type.type, type.id)); + throw IllegalArgumentException(make_string("Unknown datatype type %d for id %d", type.type, type.id)); } } -void addDataTypes(const vector<Datatype> &types, Repo &repo, - const AnnotationTypeRepo &a_repo) { +void addDataTypes(const vector<Datatype> &types, Repo &repo, const AnnotationTypeRepo &a_repo) { for (size_t i = 0; i < types.size(); ++i) { addDataType(types[i], repo, a_repo); } } void addDocumentTypes(const DocumentTypeMap &type_map, Repo &repo) { - for (DocumentTypeMap::const_iterator - it = type_map.begin(); it != type_map.end(); ++it) { - repo.addDataType(*it->second->doc_type); + for (const auto & entry : type_map) { + repo.addDataType(*entry.second->doc_type); } } -void addDefaultDocument(DocumentTypeMap &type_map) { +const DocumentType * +addDefaultDocument(DocumentTypeMap &type_map) { DataTypeRepo::UP data_types(new DataTypeRepo); vector<const DataType *> default_types = DataType::getDefaultDataTypes(); for (size_t i = 0; i < default_types.size(); ++i) { @@ -385,97 +360,85 @@ void addDefaultDocument(DocumentTypeMap &type_map) { data_types->repo.addDataType(PositionDataType::getInstance()); data_types->doc_type = new DocumentType("document", 8); - vector<const AnnotationType *> annotation_types( - AnnotationType::getDefaultAnnotationTypes()); + vector<const AnnotationType *> annotation_types(AnnotationType::getDefaultAnnotationTypes()); for(size_t i(0); i < annotation_types.size(); ++i) { - data_types->annotations.addAnnotationType( - AnnotationType::UP(new AnnotationType(*annotation_types[i]))); + data_types->annotations.addAnnotationType(std::make_unique<AnnotationType>(*annotation_types[i])); } uint32_t typeId = data_types->doc_type->getId(); + const DocumentType * docType = data_types->doc_type; type_map[typeId] = data_types.release(); + return docType; } const DataTypeRepo &lookupRepo(int32_t id, const DocumentTypeMap &type_map) { DocumentTypeMap::const_iterator it = type_map.find(id); if (it == type_map.end()) { - throw IllegalArgumentException( - make_string("Unable to find document type %d.", id)); + throw IllegalArgumentException(make_string("Unable to find document type %d.", id)); } return *it->second; } -void inheritDataTypes( - const vector<DocumenttypesConfig::Documenttype::Inherits> &base_types, - const DocumentTypeMap &type_map, Repo &repo) { +void inheritDataTypes(const vector<DocumenttypesConfig::Documenttype::Inherits> &base_types, + const DocumentTypeMap &type_map, Repo &repo) { repo.inherit(lookupRepo(DataType::T_DOCUMENT, type_map).repo); for (size_t i = 0; i < base_types.size(); ++i) { repo.inherit(lookupRepo(base_types[i].id, type_map).repo); } } -void inheritAnnotationTypes( - const vector<DocumenttypesConfig::Documenttype::Inherits> &base_types, - const DocumentTypeMap &type_map, AnnotationTypeRepo &repo) { +void inheritAnnotationTypes(const vector<DocumenttypesConfig::Documenttype::Inherits> &base_types, + const DocumentTypeMap &type_map, AnnotationTypeRepo &repo) { repo.inherit(lookupRepo(DataType::T_DOCUMENT, type_map).annotations); for (size_t i = 0; i < base_types.size(); ++i) { repo.inherit(lookupRepo(base_types[i].id, type_map).annotations); } } -void inheritDocumentTypes( - const vector<DocumenttypesConfig::Documenttype::Inherits> &base_types, - const DocumentTypeMap &type_map, DocumentType &doc_type) { +void inheritDocumentTypes(const vector<DocumenttypesConfig::Documenttype::Inherits> &base_types, + const DocumentTypeMap &type_map, DocumentType &doc_type) { for (size_t i = 0; i < base_types.size(); ++i) { const DataTypeRepo &parent = lookupRepo(base_types[i].id, type_map); doc_type.inherit(*parent.doc_type); } } -DataTypeRepo::UP makeDataTypeRepo( - const DocumentType &doc_type, - const DocumentTypeMap &type_map) { +DataTypeRepo::UP makeDataTypeRepo(const DocumentType &doc_type, const DocumentTypeMap &type_map) { DataTypeRepo::UP data_types(new DataTypeRepo); data_types->repo.inherit(lookupRepo(DataType::T_DOCUMENT, type_map).repo); - data_types->annotations.inherit( - lookupRepo(DataType::T_DOCUMENT, type_map).annotations); + data_types->annotations.inherit(lookupRepo(DataType::T_DOCUMENT, type_map).annotations); data_types->doc_type = doc_type.clone(); return data_types; } void addFieldSet(const DocumenttypesConfig::Documenttype::FieldsetsMap & fsv, DocumentType &doc_type) { - for (DocumenttypesConfig::Documenttype::FieldsetsMap::const_iterator it(fsv.begin()), mt(fsv.end()); it != mt; it++) { - const DocumenttypesConfig::Documenttype::Fieldsets & fs(it->second); + for (const auto & entry : fsv) { + const DocumenttypesConfig::Documenttype::Fieldsets & fs(entry.second); DocumentType::FieldSet::Fields fields; for (size_t j(0); j < fs.fields.size(); j++) { fields.insert(fs.fields[j]); } - doc_type.addFieldSet(it->first, fields); + doc_type.addFieldSet(entry.first, fields); } } -void addReferenceTypes( - const vector<DocumenttypesConfig::Documenttype::Referencetype> &ref_types, - Repo& data_type_repo, - const DocumentTypeMap& doc_type_map) { +void addReferenceTypes(const vector<DocumenttypesConfig::Documenttype::Referencetype> &ref_types, + Repo& data_type_repo, const DocumentTypeMap& doc_type_map) +{ for (const auto& ref_type : ref_types) { const auto* target_doc_type = lookupRepo(ref_type.targetTypeId, doc_type_map).doc_type; data_type_repo.addDataType(std::make_unique<ReferenceDataType>(*target_doc_type, ref_type.id)); } } -void configureDataTypeRepo( - const DocumenttypesConfig::Documenttype &doc_type, - DocumentTypeMap &type_map) { +void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, DocumentTypeMap &type_map) { DataTypeRepo *data_types = type_map[doc_type.id]; - inheritAnnotationTypes( - doc_type.inherits, type_map, data_types->annotations); + inheritAnnotationTypes(doc_type.inherits, type_map, data_types->annotations); addAnnotationTypes(doc_type.annotationtype, data_types->annotations); inheritDataTypes(doc_type.inherits, type_map, data_types->repo); addReferenceTypes(doc_type.referencetype, data_types->repo, type_map); addDataTypes(doc_type.datatype, data_types->repo, data_types->annotations); - setAnnotationDataTypes(doc_type.annotationtype, data_types->annotations, - data_types->repo); + setAnnotationDataTypes(doc_type.annotationtype, data_types->annotations, data_types->repo); inheritDocumentTypes(doc_type.inherits, type_map, *data_types->doc_type); addFieldSet(doc_type.fieldsets, *data_types->doc_type); } @@ -483,39 +446,33 @@ void configureDataTypeRepo( void addDataTypeRepo(DataTypeRepo::UP data_types, DocumentTypeMap &doc_types) { DataTypeRepo *& p = doc_types[data_types->doc_type->getId()]; if (p) { - LOG(warning, "Type repo already exists for id %d.", - data_types->doc_type->getId()); + LOG(warning, "Type repo already exists for id %d.", data_types->doc_type->getId()); throw IllegalArgumentException("Trying to redefine a document type."); } p = data_types.release(); } -DataTypeRepo::UP makeSkeletonDataTypeRepo( - const DocumenttypesConfig::Documenttype &type) { +DataTypeRepo::UP makeSkeletonDataTypeRepo(const DocumenttypesConfig::Documenttype &type) { DataTypeRepo::UP data_types(new DataTypeRepo); - StructDataType::UP - type_ap(new StructDataType(type.name + ".header", type.headerstruct)); + auto type_ap = std::make_unique<StructDataType>(type.name + ".header", type.headerstruct); data_types->doc_type = new DocumentType(type.name, type.id, *type_ap); data_types->repo.addDataType(std::move(type_ap)); return data_types; } -void createAllDocumentTypes(const DocumenttypesConfig::DocumenttypeVector &t, - DocumentTypeMap &type_map) { +void createAllDocumentTypes(const DocumenttypesConfig::DocumenttypeVector &t, DocumentTypeMap &type_map) { for (size_t i = 0; i < t.size(); ++i) { addDataTypeRepo(makeSkeletonDataTypeRepo(t[i]), type_map); } } void addAllDocumentTypesToRepos(DocumentTypeMap &type_map) { - for (DocumentTypeMap::const_iterator - it = type_map.begin(); it != type_map.end(); ++it) { - addDocumentTypes(type_map, it->second->repo); + for (const auto & entry : type_map) { + addDocumentTypes(type_map, entry.second->repo); } } -void configureAllRepos(const DocumenttypesConfig::DocumenttypeVector &t, - DocumentTypeMap &type_map) { +void configureAllRepos(const DocumenttypesConfig::DocumenttypeVector &t, DocumentTypeMap &type_map) { for (size_t i = 0; i < t.size(); ++i) { configureDataTypeRepo(t[i], type_map); } @@ -524,15 +481,15 @@ void configureAllRepos(const DocumenttypesConfig::DocumenttypeVector &t, } // namespace DocumentTypeRepo::DocumentTypeRepo() : - _doc_types(std::make_unique<internal::DocumentTypeMap>()) + _doc_types(std::make_unique<internal::DocumentTypeMap>()), + _default(addDefaultDocument(*_doc_types)) { - addDefaultDocument(*_doc_types); } DocumentTypeRepo::DocumentTypeRepo(const DocumentType & type) : - _doc_types(std::make_unique<internal::DocumentTypeMap>()) + _doc_types(std::make_unique<internal::DocumentTypeMap>()), + _default(addDefaultDocument(*_doc_types)) { - addDefaultDocument(*_doc_types); try { addDataTypeRepo(makeDataTypeRepo(type, *_doc_types), *_doc_types); } catch (...) { @@ -542,9 +499,9 @@ DocumentTypeRepo::DocumentTypeRepo(const DocumentType & type) : } DocumentTypeRepo::DocumentTypeRepo(const DocumenttypesConfig &config) : - _doc_types(std::make_unique<internal::DocumentTypeMap>()) + _doc_types(std::make_unique<internal::DocumentTypeMap>()), + _default(addDefaultDocument(*_doc_types)) { - addDefaultDocument(*_doc_types); try { createAllDocumentTypes(config.documenttype, *_doc_types); addAllDocumentTypesToRepos(*_doc_types); @@ -559,14 +516,15 @@ DocumentTypeRepo::~DocumentTypeRepo() { DeleteMapContent(*_doc_types); } -const DocumentType *DocumentTypeRepo::getDocumentType(int32_t type_id) const { +const DocumentType * +DocumentTypeRepo::getDocumentType(int32_t type_id) const { const DataTypeRepo *repo = FindPtr(*_doc_types, type_id); return repo ? repo->doc_type : nullptr; } -const DocumentType *DocumentTypeRepo::getDocumentType(const stringref &name) const { - DocumentTypeMap::const_iterator it = - _doc_types->find(DocumentType::createId(name)); +const DocumentType * +DocumentTypeRepo::getDocumentType(const stringref &name) const { + DocumentTypeMap::const_iterator it = _doc_types->find(DocumentType::createId(name)); if (it != _doc_types->end() && it->second->doc_type->getName() == name) { return it->second->doc_type; @@ -586,23 +544,21 @@ DocumentTypeRepo::getDataType(const DocumentType &doc_type, int32_t id) const { } const DataType * -DocumentTypeRepo::getDataType( - const DocumentType &doc_type, const stringref &name) const { +DocumentTypeRepo::getDataType(const DocumentType &doc_type, const stringref &name) const { const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId()); return dt_repo ? dt_repo->repo.lookup(name) : nullptr; } -const AnnotationType *DocumentTypeRepo::getAnnotationType( - const DocumentType &doc_type, int32_t id) const { +const AnnotationType * +DocumentTypeRepo::getAnnotationType(const DocumentType &doc_type, int32_t id) const { const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId()); return dt_repo ? dt_repo->annotations.lookup(id) : nullptr; } -void DocumentTypeRepo::forEachDocumentType( - Closure1<const DocumentType &> &c) const { - for (DocumentTypeMap::const_iterator - it = _doc_types->begin(); it != _doc_types->end(); ++it) { - c.call(*it->second->doc_type); +void +DocumentTypeRepo::forEachDocumentType(Closure1<const DocumentType &> &c) const { + for (const auto & entry : *_doc_types) { + c.call(*entry.second->doc_type); } } diff --git a/document/src/vespa/document/repo/documenttyperepo.h b/document/src/vespa/document/repo/documenttyperepo.h index 71410197405..c1c25204b3f 100644 --- a/document/src/vespa/document/repo/documenttyperepo.h +++ b/document/src/vespa/document/repo/documenttyperepo.h @@ -20,6 +20,7 @@ class DocumentType; class DocumentTypeRepo { std::unique_ptr<internal::DocumentTypeMap> _doc_types; + const DocumentType * _default; public: using DocumenttypesConfig = const internal::InternalDocumenttypesType; @@ -39,7 +40,7 @@ public: const DataType *getDataType(const DocumentType &doc_type, const vespalib::stringref &name) const; const AnnotationType *getAnnotationType(const DocumentType &doc_type, int32_t id) const; void forEachDocumentType(vespalib::Closure1<const DocumentType &> &c) const; - + const DocumentType *getDefaultDocType() const { return _default; } }; } // namespace document diff --git a/document/src/vespa/document/repo/fixedtyperepo.cpp b/document/src/vespa/document/repo/fixedtyperepo.cpp index 7644dddb64d..81a26265830 100644 --- a/document/src/vespa/document/repo/fixedtyperepo.cpp +++ b/document/src/vespa/document/repo/fixedtyperepo.cpp @@ -1,38 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "fixedtyperepo.h" -#include "documenttyperepo.h" #include <cassert> namespace document { -FixedTypeRepo::FixedTypeRepo(const DocumentTypeRepo &repo) - : _repo(&repo), _doc_type(repo.getDocumentType(DataType::T_DOCUMENT)) +FixedTypeRepo::FixedTypeRepo(const DocumentTypeRepo &repo, const vespalib::string &type) + : _repo(&repo), _doc_type(repo.getDocumentType(type)) { -} - -FixedTypeRepo::FixedTypeRepo(const DocumentTypeRepo &repo, - const vespalib::string &type) - : _repo(&repo), _doc_type(repo.getDocumentType(type)) { assert(_doc_type); } -const DataType * -FixedTypeRepo::getDataType(int32_t id) const -{ - return _repo->getDataType(*_doc_type, id); -} - -const DataType * -FixedTypeRepo::getDataType(const vespalib::string &name) const -{ - return _repo->getDataType(*_doc_type, name); -} - -const AnnotationType * -FixedTypeRepo::getAnnotationType(int32_t id) const -{ - return _repo->getAnnotationType(*_doc_type, id); -} - } // namespace document diff --git a/document/src/vespa/document/repo/fixedtyperepo.h b/document/src/vespa/document/repo/fixedtyperepo.h index eb8a5a328dd..67e7571e31d 100644 --- a/document/src/vespa/document/repo/fixedtyperepo.h +++ b/document/src/vespa/document/repo/fixedtyperepo.h @@ -2,13 +2,10 @@ #pragma once -#include <vespa/document/datatype/datatype.h> +#include "documenttyperepo.h" namespace document { -class DocumentTypeRepo; -class AnnotationType; - // Combines a DocumentTypeRepo and a DocumentType to allow easy access // to the types contained in the DocumentType's namespace. class FixedTypeRepo { @@ -16,15 +13,15 @@ class FixedTypeRepo { const DocumentType *_doc_type; public: - explicit FixedTypeRepo(const DocumentTypeRepo &repo); + explicit FixedTypeRepo(const DocumentTypeRepo &repo) + : _repo(&repo), _doc_type(repo.getDefaultDocType()) {} FixedTypeRepo(const DocumentTypeRepo &repo, const DocumentType &doc_type) : _repo(&repo), _doc_type(&doc_type) {} FixedTypeRepo(const DocumentTypeRepo &repo, const vespalib::string &type); - const DataType *getDataType(int32_t id) const; - const DataType *getDataType(const vespalib::string &name) const; - const AnnotationType *getAnnotationType(int32_t id) const; - + const DataType *getDataType(int32_t id) const { return _repo->getDataType(*_doc_type, id); } + const DataType *getDataType(const vespalib::string &name) const { return _repo->getDataType(*_doc_type, name); } + const AnnotationType *getAnnotationType(int32_t id) const { return _repo->getAnnotationType(*_doc_type, id); } const DocumentTypeRepo &getDocumentTypeRepo() const { return *_repo; } const DocumentType &getDocumentType() const { return *_doc_type; } }; diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 1a628ea2618..08fddbaad41 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -425,7 +425,7 @@ namespace { void VespaDocumentSerializer::write42(const DocumentUpdate &value) { - _stream << static_cast<uint16_t>(value.getVersion()); + _stream << static_cast<uint16_t>(Document::getNewestSerializationVersion()); write(value.getId()); _stream << static_cast<uint8_t>(CONTENT_HASTYPE); _stream.write(value.getType().getName().c_str(), value.getType().getName().size() + 1); @@ -439,17 +439,19 @@ void VespaDocumentSerializer::write42(const DocumentUpdate &value) void VespaDocumentSerializer::writeHEAD(const DocumentUpdate &value) { + if (!value._needHardReserialize) { + _stream.write(value._backing.peek(), value._backing.size()); + return; + } write(value.getId()); _stream.write(value.getType().getName().c_str(), value.getType().getName().size() + 1); _stream << static_cast<uint16_t>(0); - const DocumentUpdate::FieldUpdateV & updates(value.getUpdates()); - _stream << static_cast<uint32_t>(updates.size()); - for (const auto & update : updates) { + _stream << static_cast<uint32_t>(value._updates.size()); + for (const auto & update : value._updates) { write(update); } - const DocumentUpdate::FieldPathUpdateV & fieldPathUpdates(value.getFieldPathUpdates()); - _stream << static_cast<uint32_t>(value.serializeFlags(fieldPathUpdates.size())); - for (const auto & update : fieldPathUpdates) { + _stream << static_cast<uint32_t>(value.serializeFlags(value._fieldPathUpdates.size())); + for (const auto & update : value._fieldPathUpdates) { _stream << update->getSerializedType(); write(*update); } diff --git a/document/src/vespa/document/update/addfieldpathupdate.cpp b/document/src/vespa/document/update/addfieldpathupdate.cpp index d0796b9b2a0..ab719de38d4 100644 --- a/document/src/vespa/document/update/addfieldpathupdate.cpp +++ b/document/src/vespa/document/update/addfieldpathupdate.cpp @@ -3,8 +3,8 @@ #include "addfieldpathupdate.h" #include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <ostream> @@ -88,10 +88,9 @@ AddFieldPathUpdate::print(std::ostream& out, bool verbose, const std::string& in } void -AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) +AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) { - FieldPathUpdate::deserialize(repo, type, buffer, version); + FieldPathUpdate::deserialize(repo, type, stream); FieldPath path; type.buildFieldPath(path, getOriginalFieldPath()); @@ -99,10 +98,8 @@ AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& ty assert(fieldType.inherits(ArrayDataType::classId)); FieldValue::UP val = fieldType.createFieldValue(); _values.reset(static_cast<ArrayFieldValue*>(val.release())); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - VespaDocumentDeserializer deserializer(repo, stream, version); + VespaDocumentDeserializer deserializer(repo, stream, Document::getNewestSerializationVersion()); deserializer.read(*_values); - buffer.incPos(buffer.getRemaining() - stream.size()); } std::unique_ptr<IteratorHandler> diff --git a/document/src/vespa/document/update/addfieldpathupdate.h b/document/src/vespa/document/update/addfieldpathupdate.h index 2943248d9d8..692a30e3a73 100644 --- a/document/src/vespa/document/update/addfieldpathupdate.h +++ b/document/src/vespa/document/update/addfieldpathupdate.h @@ -26,8 +26,7 @@ public: private: uint8_t getSerializedType() const override { return AddMagic; } - void deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) override; + void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) override; std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document &, const DocumentTypeRepo &) const override; diff --git a/document/src/vespa/document/update/addvalueupdate.cpp b/document/src/vespa/document/update/addvalueupdate.cpp index 8ac1ee301eb..051ffcf8b2a 100644 --- a/document/src/vespa/document/update/addvalueupdate.cpp +++ b/document/src/vespa/document/update/addvalueupdate.cpp @@ -5,14 +5,13 @@ #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/document/util/serializableexceptions.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/xmlstream.h> - using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; using vespalib::nbostream; +using vespalib::make_string; using namespace vespalib::xml; namespace document { @@ -25,7 +24,7 @@ AddValueUpdate:: AddValueUpdate(const FieldValue& value, int weight) _weight(weight) {} -AddValueUpdate::~AddValueUpdate() { } +AddValueUpdate::~AddValueUpdate() = default; bool AddValueUpdate::operator==(const ValueUpdate& other) const { @@ -41,19 +40,14 @@ void AddValueUpdate::checkCompatibility(const Field& field) const { if (field.getDataType().inherits(CollectionDataType::classId)) { - const CollectionDataType& type( - static_cast<const CollectionDataType&>(field.getDataType())); + const CollectionDataType& type(static_cast<const CollectionDataType&>(field.getDataType())); if (!type.getNestedType().isValueType(*_value)) { - throw IllegalArgumentException( - "Cannot add value of type " - + _value->getDataType()->toString() + " to field " - + field.getName().c_str() + " of container type " - + field.getDataType().toString(), VESPA_STRLOC); + throw IllegalArgumentException("Cannot add value of type " + _value->getDataType()->toString() + + " to field " + field.getName().c_str() + " of container type " + + field.getDataType().toString(), VESPA_STRLOC); } } else { - throw IllegalArgumentException( - "Can not add a value to field of type" - + field.getDataType().toString(), VESPA_STRLOC); + throw IllegalArgumentException("Can not add a value to field of type" + field.getDataType().toString(), VESPA_STRLOC); } } @@ -75,9 +69,7 @@ AddValueUpdate::applyTo(FieldValue& value) const WeightedSetFieldValue& doc(static_cast<WeightedSetFieldValue&>(value)); doc.add(*_value, _weight); } else { - std::string err = vespalib::make_string( - "Unable to add a value to a \"%s\" field value.", - value.getClass().name()); + std::string err = make_string("Unable to add a value to a \"%s\" field value.", value.getClass().name()); throw IllegalStateException(err, VESPA_STRLOC); } return true; @@ -93,23 +85,16 @@ AddValueUpdate::printXml(XmlOutputStream& xos) const // Deserialize this update from the given buffer. void -AddValueUpdate::deserialize(const DocumentTypeRepo& repo, - const DataType& type, - ByteBuffer& buffer, uint16_t version) +AddValueUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream& stream) { - const CollectionDataType* ctype = - Identifiable::cast<const CollectionDataType*>(&type); - if (ctype == NULL) { - throw DeserializeException("Can not perform add operation on " - "non-collection type."); + const CollectionDataType* ctype = Identifiable::cast<const CollectionDataType*>(&type); + if (ctype == nullptr) { + throw DeserializeException("Can not perform add operation on non-collection type."); } _value.reset(ctype->getNestedType().createFieldValue().release()); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - VespaDocumentDeserializer deserializer(repo, stream, version); + VespaDocumentDeserializer deserializer(repo, stream, Document::getNewestSerializationVersion()); deserializer.read(*_value); - buffer.incPos(buffer.getRemaining() - stream.size()); - buffer.getIntNetwork(_weight); + stream >> _weight; } } - diff --git a/document/src/vespa/document/update/addvalueupdate.h b/document/src/vespa/document/update/addvalueupdate.h index 439263c5089..015d517c5a1 100644 --- a/document/src/vespa/document/update/addvalueupdate.h +++ b/document/src/vespa/document/update/addvalueupdate.h @@ -7,8 +7,8 @@ */ #pragma once +#include "valueupdate.h" #include <vespa/document/fieldvalue/fieldvalue.h> -#include <vespa/document/update/valueupdate.h> namespace document { @@ -62,18 +62,14 @@ public: return *this; } - // ValueUpdate implementation void checkCompatibility(const Field& field) const override; bool applyTo(FieldValue& value) const override; void printXml(XmlOutputStream& xos) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) override; + void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & buffer) override; AddValueUpdate* clone() const override { return new AddValueUpdate(*this); } DECLARE_IDENTIFIABLE(AddValueUpdate); - }; } // document - diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.cpp b/document/src/vespa/document/update/arithmeticvalueupdate.cpp index b84dbe0f366..9ae7dd17a52 100644 --- a/document/src/vespa/document/update/arithmeticvalueupdate.cpp +++ b/document/src/vespa/document/update/arithmeticvalueupdate.cpp @@ -2,7 +2,7 @@ #include "arithmeticvalueupdate.h" #include <vespa/document/base/field.h> #include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/xmlstream.h> @@ -127,14 +127,11 @@ ArithmeticValueUpdate::printXml(XmlOutputStream& xos) const // Deserialize this update from the given buffer. void -ArithmeticValueUpdate::deserialize( - const DocumentTypeRepo&, const DataType&, - ByteBuffer& buffer, uint16_t) +ArithmeticValueUpdate::deserialize(const DocumentTypeRepo&, const DataType&, nbostream & stream) { int32_t opt; - buffer.getIntNetwork(opt); + stream >> opt >>_operand; _operator = static_cast<ArithmeticValueUpdate::Operator>(opt); - buffer.getDoubleNetwork(_operand); } } // document diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.h b/document/src/vespa/document/update/arithmeticvalueupdate.h index 60718f8ad1f..1f9e33a3c3a 100644 --- a/document/src/vespa/document/update/arithmeticvalueupdate.h +++ b/document/src/vespa/document/update/arithmeticvalueupdate.h @@ -8,7 +8,7 @@ */ #pragma once -#include <vespa/document/update/valueupdate.h> +#include "valueupdate.h" namespace document { @@ -57,10 +57,7 @@ public: bool operator==(const ValueUpdate& other) const override; - /** @return the operator of this arithmetic update. */ Operator getOperator() const { return _operator; } - - /** @return the operand of this arithmetic update. */ double getOperand() const { return _operand; } /** @@ -87,17 +84,14 @@ public: */ long applyTo(int64_t value) const; - // ValueUpdate implementation void checkCompatibility(const Field& field) const override; bool applyTo(FieldValue& value) const override; void printXml(XmlOutputStream& xos) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) override; + void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & buffer) override; ArithmeticValueUpdate* clone() const override { return new ArithmeticValueUpdate(*this); } DECLARE_IDENTIFIABLE(ArithmeticValueUpdate); - }; } // document diff --git a/document/src/vespa/document/update/assignfieldpathupdate.cpp b/document/src/vespa/document/update/assignfieldpathupdate.cpp index 1de38b982d0..bec717874dc 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.cpp +++ b/document/src/vespa/document/update/assignfieldpathupdate.cpp @@ -6,7 +6,6 @@ #include <vespa/document/select/parser.h> #include <vespa/document/select/variablemap.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <boost/numeric/conversion/cast.hpp> @@ -218,27 +217,24 @@ AssignFieldPathUpdate::print(std::ostream& out, bool verbose, const std::string& } void -AssignFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) +AssignFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) { - FieldPathUpdate::deserialize(repo, type, buffer, version); + FieldPathUpdate::deserialize(repo, type, stream); uint8_t flags = 0x00; - buffer.getByte(flags); + stream >> flags; _removeIfZero = (flags & REMOVE_IF_ZERO) != 0; _createMissingPath = (flags & CREATE_MISSING_PATH) != 0; if (flags & ARITHMETIC_EXPRESSION) { - _expression = getString(buffer); + _expression = getString(stream); } else { FieldPath path; type.buildFieldPath(path, getOriginalFieldPath()); _newValue.reset(getResultingDataType(path).createFieldValue().release()); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - VespaDocumentDeserializer deserializer(repo, stream, version); + VespaDocumentDeserializer deserializer(repo, stream, Document::getNewestSerializationVersion()); deserializer.read(*_newValue); - buffer.incPos(buffer.getRemaining() - stream.size()); } } diff --git a/document/src/vespa/document/update/assignfieldpathupdate.h b/document/src/vespa/document/update/assignfieldpathupdate.h index 329ce5d8c93..a1349bab96a 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.h +++ b/document/src/vespa/document/update/assignfieldpathupdate.h @@ -44,7 +44,7 @@ public: private: uint8_t getSerializedType() const override { return AssignMagic; } - void deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) override; + void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) override; std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document& doc, const DocumentTypeRepo & repo) const override; @@ -54,4 +54,4 @@ private: bool _createMissingPath; }; -} // ns document +} diff --git a/document/src/vespa/document/update/assignvalueupdate.cpp b/document/src/vespa/document/update/assignvalueupdate.cpp index fb38a78690b..0f7cf243ba3 100644 --- a/document/src/vespa/document/update/assignvalueupdate.cpp +++ b/document/src/vespa/document/update/assignvalueupdate.cpp @@ -4,7 +4,6 @@ #include <vespa/document/base/field.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/xmlstream.h> @@ -90,19 +89,17 @@ AssignValueUpdate::printXml(XmlOutputStream& xos) const // Deserialize this update from the given buffer. void -AssignValueUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) +AssignValueUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) { // Read content bit vector. - unsigned char content = 0x00; - buffer.getByte(content); + uint8_t content = 0x00; + stream >> content; // Read field value, if any. if (content & CONTENT_HASVALUE) { _value.reset(type.createFieldValue().release()); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - VespaDocumentDeserializer deserializer(repo, stream, version); + VespaDocumentDeserializer deserializer(repo, stream, Document::getNewestSerializationVersion()); deserializer.read(*_value); - buffer.incPos(buffer.getRemaining() - stream.size()); } } diff --git a/document/src/vespa/document/update/assignvalueupdate.h b/document/src/vespa/document/update/assignvalueupdate.h index 05802857f65..978e61cbb3c 100644 --- a/document/src/vespa/document/update/assignvalueupdate.h +++ b/document/src/vespa/document/update/assignvalueupdate.h @@ -26,34 +26,26 @@ public: AssignValueUpdate(); AssignValueUpdate(const FieldValue& value); - ~AssignValueUpdate(); + ~AssignValueUpdate() override; bool operator==(const ValueUpdate& other) const override; - /** @return The field value to assign during this update. */ bool hasValue() const { return bool(_value); } const FieldValue& getValue() const { return *_value; } - const FieldValue* getValuePtr() const { return _value.get(); } - /** - * Sets the field value to assign during this update. - * @return A reference to this. - */ AssignValueUpdate& setValue(const FieldValue* value) { _value.reset(value ? value->clone() : 0); return *this; } - // ValueUpdate implementation. void checkCompatibility(const Field& field) const override; bool applyTo(FieldValue& value) const override; void printXml(XmlOutputStream& xos) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) override; + void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & buffer) override; AssignValueUpdate* clone() const override { return new AssignValueUpdate(*this); } DECLARE_IDENTIFIABLE(AssignValueUpdate); }; -} // document +} diff --git a/document/src/vespa/document/update/clearvalueupdate.cpp b/document/src/vespa/document/update/clearvalueupdate.cpp index 0a69881181a..e2b8dc86afd 100644 --- a/document/src/vespa/document/update/clearvalueupdate.cpp +++ b/document/src/vespa/document/update/clearvalueupdate.cpp @@ -1,14 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "clearvalueupdate.h" -#include <vespa/document/base/field.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/xmlstream.h> #include <ostream> -using vespalib::IllegalArgumentException; -using vespalib::IllegalStateException; using namespace vespalib::xml; namespace document { @@ -29,9 +24,8 @@ ClearValueUpdate::checkCompatibility(const Field&) const // Apply this update to the given document. bool -ClearValueUpdate::applyTo(FieldValue& value) const +ClearValueUpdate::applyTo(FieldValue& ) const { - (void) value; return false; } @@ -50,8 +44,7 @@ ClearValueUpdate::print(std::ostream& out, bool, const std::string&) const // Deserialize this update from the given buffer. void -ClearValueUpdate::deserialize(const DocumentTypeRepo&, const DataType&, - ByteBuffer&, uint16_t) +ClearValueUpdate::deserialize(const DocumentTypeRepo&, const DataType&, nbostream &) { } diff --git a/document/src/vespa/document/update/clearvalueupdate.h b/document/src/vespa/document/update/clearvalueupdate.h index 12f2b888ea1..2ac47deaaf9 100644 --- a/document/src/vespa/document/update/clearvalueupdate.h +++ b/document/src/vespa/document/update/clearvalueupdate.h @@ -19,17 +19,15 @@ public: ClearValueUpdate(const ClearValueUpdate& update) : ValueUpdate(update) {} bool operator==(const ValueUpdate& other) const override; - // ValueUpdate implementation void checkCompatibility(const Field& field) const override; bool applyTo(FieldValue& value) const override; void printXml(XmlOutputStream& xos) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) override; + void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream& buffer) override; ClearValueUpdate* clone() const override { return new ClearValueUpdate(*this); } DECLARE_IDENTIFIABLE(ClearValueUpdate); }; -} // document +} diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index 30b760a6102..d23165cc815 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -22,62 +22,96 @@ using namespace vespalib::xml; namespace document { -// Declare content bits. -static const unsigned char CONTENT_HASTYPE = 0x01; +namespace { + +constexpr unsigned char CONTENT_HASTYPE = 0x01; + +vespalib::stringref +readCStr(nbostream & stream) { + const char * s = stream.peek(); + size_t sz = strnlen(s, stream.size()); + stream.adjustReadPos(sz+1); + return vespalib::stringref(s, sz); +} + +std::pair<const DocumentType *, DocumentId> +deserializeTypeAndId(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { + DocumentId docId(readCStr(stream)); + + // Read content bit vector. + unsigned char content = 0x00; + stream >> content; + + // Why on earth do we have this whether we have type part? + // We need type for object to work, so just throwing exception if it's + // not there. + if((content & CONTENT_HASTYPE) == 0) { + throw IllegalStateException("Missing document type", VESPA_STRLOC); + } + + vespalib::stringref typestr = readCStr(stream); + + int16_t version = 0; + stream >> version; + const DocumentType *type = repo.getDocumentType(typestr); + if (!type) { + throw DocumentTypeNotFoundException(typestr, VESPA_STRLOC); + } + return std::make_pair(type, docId); +} + +const DocumentType * +deserializeHeader(const DocumentTypeRepo &repo, vespalib::nbostream & stream, vespalib::stringref & documentId) +{ + documentId = readCStr(stream); + vespalib::stringref typestr = readCStr(stream); + int16_t version = 0; + stream >> version; + const DocumentType * docType = repo.getDocumentType(typestr); + if (!docType) { + throw DocumentTypeNotFoundException(typestr, VESPA_STRLOC); + } + return docType; +} + +} + -DocumentUpdate::DocumentUpdate(const DataType &type, const DocumentId& id) +DocumentUpdate::DocumentUpdate(const DocumentTypeRepo & repo, const DataType &type, const DocumentId& id) : _documentId(id), _type(&type), + _repo(&repo), + _backing(), _updates(), _fieldPathUpdates(), - _version(Document::getNewestSerializationVersion()), - _createIfNonExistent(false) + _createIfNonExistent(false), + _needHardReserialize(false) { if (!type.getClass().inherits(DocumentType::classId)) { - throw IllegalArgumentException("Cannot generate a document with non-document type " + type.toString() + ".", - VESPA_STRLOC); + throw IllegalArgumentException("Cannot generate a document with non-document type " + type.toString() + ".", VESPA_STRLOC); } + serializeHeader(); } -DocumentUpdate::DocumentUpdate(const DocumentTypeRepo& repo, - ByteBuffer& buffer, - SerializeVersion serializeVersion) - : _documentId("doc::"), +DocumentUpdate::DocumentUpdate() + : _documentId(), _type(DataType::DOCUMENT), + _repo(nullptr), + _backing(), _updates(), - _version(Document::getNewestSerializationVersion()), - _createIfNonExistent(false) + _fieldPathUpdates(), + _createIfNonExistent(false), + _needHardReserialize(false) { - switch (serializeVersion) { - case SerializeVersion::SERIALIZE_HEAD: - deserializeHEAD(repo, buffer); - break; - case SerializeVersion::SERIALIZE_42: - deserialize42(repo, buffer); - break; - default: - throw IllegalArgumentException("bad serializeVersion provided.", VESPA_STRLOC); - } } DocumentUpdate::~DocumentUpdate() = default; - bool DocumentUpdate::operator==(const DocumentUpdate& other) const { - if (_documentId != other._documentId) return false; - if (*_type != *other._type) return false; - if (_updates.size() != other._updates.size()) return false; - for (std::size_t i = 0, n = _updates.size(); i < n; ++i) { - if (_updates[i] != other._updates[i]) return false; - } - if (_fieldPathUpdates.size() != other._fieldPathUpdates.size()) return false; - for (std::size_t i = 0, n = _fieldPathUpdates.size(); i < n; ++i) { - if (*_fieldPathUpdates[i] != *other._fieldPathUpdates[i]) return false; - } - if (_createIfNonExistent != other._createIfNonExistent) return false; - return true; + return (_backing.size() == other._backing.size()) && + (memcmp(_backing.peek(), other._backing.peek(), _backing.size()) == 0); } const DocumentType& @@ -85,22 +119,69 @@ DocumentUpdate::getType() const { return static_cast<const DocumentType &> (*_type); } +const DocumentUpdate::FieldUpdateV & +DocumentUpdate::getUpdates() const { + ensureDeserialized(); + return _updates; +} + +const DocumentUpdate::FieldPathUpdateV & +DocumentUpdate::getFieldPathUpdates() const { + ensureDeserialized(); + return _fieldPathUpdates; +} + +void +DocumentUpdate::eagerDeserialize() const { + ensureDeserialized(); +} + +void DocumentUpdate::lazyDeserialize(const DocumentTypeRepo & repo, nbostream & stream) { + size_t start(stream.rp()); + vespalib::stringref voidId; + deserializeHeader(repo, stream, voidId); + deserializeBody(repo, stream); + stream.rp(start); +} +void DocumentUpdate::ensureDeserialized() const { + if (_updates.empty() && _fieldPathUpdates.empty()) { + const_cast<DocumentUpdate &>(*this).lazyDeserialize(*_repo, const_cast<nbostream &>(_backing)); + } +} + DocumentUpdate& DocumentUpdate::addUpdate(const FieldUpdate& update) { + ensureDeserialized(); _updates.push_back(update); + reserialize(); return *this; } DocumentUpdate& DocumentUpdate::addFieldPathUpdate(const FieldPathUpdate::CP& update) { + ensureDeserialized(); _fieldPathUpdates.push_back(update); + reserialize(); return *this; } void -DocumentUpdate::print(std::ostream& out, bool verbose, - const std::string& indent) const +DocumentUpdate::setCreateIfNonExistent(bool value) { + ensureDeserialized(); + _createIfNonExistent = value; + reserialize(); +} + +bool +DocumentUpdate::getCreateIfNonExistent() const { + ensureDeserialized(); + return _createIfNonExistent; +} + +void +DocumentUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const { + ensureDeserialized(); out << "DocumentUpdate("; if (_type) { _type->print(out, verbose, indent + " "); @@ -131,6 +212,7 @@ DocumentUpdate::print(std::ostream& out, bool verbose, void DocumentUpdate::applyTo(Document& doc) const { + ensureDeserialized(); const DocumentType& type = doc.getType(); if (_type->getName() != type.getName()) { string err = make_string("Can not apply a \"%s\" document update to a \"%s\" document.", @@ -150,6 +232,17 @@ DocumentUpdate::applyTo(Document& doc) const } void +DocumentUpdate::serializeHeader() { + string id_string = _documentId.getScheme().toString(); + _backing.write(id_string.data(), id_string.size()); + _backing << static_cast<uint8_t>(0); + _backing.write(getType().getName().c_str(), getType().getName().size() + 1); + _backing << static_cast<uint16_t>(0); // version + _backing << static_cast<uint32_t>(0); // Number of updates + _backing << static_cast<uint32_t>(0); // Number of field path updates +} + +void DocumentUpdate::serializeHEAD(nbostream &stream) const { VespaDocumentSerializer serializer(stream); @@ -164,121 +257,122 @@ DocumentUpdate::serializeFlags(int size_) const return flags.injectInto(size_); } -namespace { - std::pair<const DocumentType *, DocumentId> - deserializeTypeAndId(const DocumentTypeRepo& repo, ByteBuffer& buffer) { - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - DocumentId docId(stream); - buffer.incPos(stream.rp()); - - // Read content bit vector. - unsigned char content = 0x00; - buffer.getByte(content); - - // Why on earth do we have this whether we have type part? - // We need type for object to work, so just throwing exception if it's - // not there. - if((content & CONTENT_HASTYPE) == 0) { - throw IllegalStateException("Missing document type", VESPA_STRLOC); - } - - vespalib::stringref typestr = buffer.getBufferAtPos(); - buffer.incPos(typestr.length() + 1); - - int16_t version = 0; - buffer.getShortNetwork(version); - const DocumentType *type = repo.getDocumentType(typestr); - if (!type) { - throw DocumentTypeNotFoundException(typestr, VESPA_STRLOC); - } - return std::make_pair(type, docId); - } -} - // Deserialize the content of the given buffer into this document update. DocumentUpdate::UP -DocumentUpdate::create42(const DocumentTypeRepo& repo, ByteBuffer& buffer) +DocumentUpdate::create42(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { - return std::make_unique<DocumentUpdate>(repo, buffer, SerializeVersion::SERIALIZE_42); + auto update = std::make_unique<DocumentUpdate>(); + update->init42(repo, stream); + return update; } DocumentUpdate::UP DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, ByteBuffer& buffer) { - return std::make_unique<DocumentUpdate>(repo, buffer, SerializeVersion::SERIALIZE_HEAD); + vespalib::nbostream is(buffer.getBufferAtPos(), buffer.getRemaining()); + auto update = std::make_unique<DocumentUpdate>(); + update->initHEAD(repo, is); + buffer.setPos(buffer.getPos() + is.rp()); + return update; +} + +DocumentUpdate::UP +DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream stream) +{ + auto update = std::make_unique<DocumentUpdate>(); + update->initHEAD(repo, std::move(stream)); + return update; } void -DocumentUpdate::deserialize42(const DocumentTypeRepo& repo, ByteBuffer& buffer) +DocumentUpdate::init42(const DocumentTypeRepo & repo, vespalib::nbostream & stream) { - int pos = buffer.getPos(); - try{ - buffer.getShortNetwork(_version); + _repo = &repo; + deserialize42(repo, stream); + reserialize(); +} +void +DocumentUpdate::initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream) +{ + _repo = &repo; + _backing = std::move(stream); + size_t startPos = _backing.rp(); + vespalib::stringref docId; + _type = deserializeHeader(repo, _backing, docId); + _documentId.set(docId); + _backing.rp(startPos); +} + +void +DocumentUpdate::initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream & stream) +{ + size_t startPos = stream.rp(); + vespalib::stringref docId; + _type = deserializeHeader(repo, stream, docId); + _documentId.set(docId); + deserializeBody(repo, stream); + size_t sz = stream.rp() - startPos; + _backing = nbostream(stream.peek() - sz, sz); +} - std::pair<const DocumentType *, DocumentId> typeAndId(deserializeTypeAndId(repo, buffer)); +void +DocumentUpdate::deserialize42(const DocumentTypeRepo& repo, vespalib::nbostream & stream) +{ + size_t pos = stream.rp(); + try{ + int16_t version(0); + stream >> version; + std::pair<const DocumentType *, DocumentId> typeAndId(deserializeTypeAndId(repo, stream)); _type = typeAndId.first; _documentId = typeAndId.second; // Read field updates, if any. - if(buffer.getRemaining() > 0) { - int sizeAndFlags = 0; - buffer.getIntNetwork(sizeAndFlags); + if (! stream.empty()) { + int32_t sizeAndFlags = 0; + stream >> sizeAndFlags; int numUpdates = deserializeFlags(sizeAndFlags); _updates.reserve(numUpdates); for (int i = 0; i < numUpdates; i++) { - _updates.emplace_back(repo, *typeAndId.first, buffer, _version); + _updates.emplace_back(repo, *typeAndId.first, stream); } } } catch (const DeserializeException &) { - buffer.setPos(pos); + stream.rp(pos); throw; } catch (const BufferOutOfBoundsException &) { - buffer.setPos(pos); + stream.rp(pos); throw; } } void -DocumentUpdate::deserializeHEAD(const DocumentTypeRepo &repo, ByteBuffer &buffer) +DocumentUpdate::deserializeBody(const DocumentTypeRepo &repo, vespalib::nbostream &stream) { - int pos = buffer.getPos(); + _updates.clear(); + _fieldPathUpdates.clear(); + size_t pos = stream.rp(); try { - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - _documentId = DocumentId(stream); - buffer.incPos(stream.rp()); - - vespalib::stringref typestr = buffer.getBufferAtPos(); - buffer.incPos(typestr.length() + 1); - - int16_t version = 0; - buffer.getShortNetwork(version); - const DocumentType *docType = repo.getDocumentType(typestr); - if (!docType) { - throw DocumentTypeNotFoundException(typestr, VESPA_STRLOC); - } - _type = docType; - // Read field updates, if any. - if (buffer.getRemaining() > 0) { - int numUpdates = 0; - buffer.getIntNetwork(numUpdates); + if ( ! stream.empty() ) { + int32_t numUpdates = 0; + stream >> numUpdates; _updates.reserve(numUpdates); for (int i = 0; i < numUpdates; i++) { - _updates.emplace_back(repo, *docType, buffer, 8); + _updates.emplace_back(repo, *_type, stream); } } // Read fieldpath updates, if any - int sizeAndFlags = 0; - buffer.getIntNetwork(sizeAndFlags); + int32_t sizeAndFlags = 0; + stream >> sizeAndFlags; int numUpdates = deserializeFlags(sizeAndFlags); _fieldPathUpdates.reserve(numUpdates); for (int i = 0; i < numUpdates; ++i) { - _fieldPathUpdates.emplace_back(FieldPathUpdate::createInstance(repo, *_type, buffer, 8).release()); + _fieldPathUpdates.emplace_back(FieldPathUpdate::createInstance(repo, *_type, stream).release()); } } catch (const DeserializeException &) { - buffer.setPos(pos); + stream.rp(pos); throw; } catch (const BufferOutOfBoundsException &) { - buffer.setPos(pos); + stream.rp(pos); throw; } } @@ -293,6 +387,7 @@ DocumentUpdate::deserializeFlags(int sizeAndFlags) void DocumentUpdate::printXml(XmlOutputStream& xos) const { + ensureDeserialized(); xos << XmlTag("document") << XmlAttribute("type", _type->getName()) << XmlAttribute("id", getId().toString()); @@ -304,4 +399,15 @@ DocumentUpdate::printXml(XmlOutputStream& xos) const xos << XmlEndTag(); } +void +DocumentUpdate::reserialize() +{ + nbostream stream; + VespaDocumentSerializer serializer(stream); + _needHardReserialize = true; + serializer.writeHEAD(*this); + _backing = std::move(stream); + _needHardReserialize = false; +} + } diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h index c6250fb6b75..a7ff138af87 100644 --- a/document/src/vespa/document/update/documentupdate.h +++ b/document/src/vespa/document/update/documentupdate.h @@ -41,16 +41,6 @@ class VespaDocumentSerializer; */ class DocumentUpdate final : public Printable, public XmlSerializable { -private: - /** - * Enum class containing the legal serialization version for - * document updates. This version is not encoded in the serialized - * document update. - */ - enum class SerializeVersion { - SERIALIZE_42, // old style format, before vespa 5.0 - SERIALIZE_HEAD // new style format, since vespa 5.0 - }; public: typedef std::unique_ptr<DocumentUpdate> UP; typedef std::shared_ptr<DocumentUpdate> SP; @@ -60,22 +50,15 @@ public: /** * Create old style document update, no support for field path updates. */ - static DocumentUpdate::UP create42(const DocumentTypeRepo&, ByteBuffer&); + static DocumentUpdate::UP create42(const DocumentTypeRepo & repo, vespalib::nbostream & stream); /** * Create new style document update, possibly with field path updates. */ - static DocumentUpdate::UP createHEAD(const DocumentTypeRepo&, ByteBuffer&); + static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream stream); + static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, ByteBuffer & buffer); - /** - * Create a document update from a byte buffer containing a serialized - * document update. Public to allow useage in std::make_unique/shared. - * - * @param repo Document type repo used to find proper document type - * @param buffer The buffer containing the serialized document update - * @param serializeVersion Selector between serialization formats. - */ - DocumentUpdate(const DocumentTypeRepo &repo, ByteBuffer &buffer, SerializeVersion serializeVersion); + DocumentUpdate(); /** * The document type is not strictly needed, as we know this at applyTo() * time, but search does not use applyTo() code to do the update, and don't @@ -86,7 +69,7 @@ public: * @param type The document type that this update is applicable for. * @param id The identifier of the document that this update is created for. */ - DocumentUpdate(const DataType &type, const DocumentId& id); + DocumentUpdate(const DocumentTypeRepo & repo, const DataType &type, const DocumentId& id); DocumentUpdate(const DocumentUpdate &) = delete; DocumentUpdate & operator = (const DocumentUpdate &) = delete; @@ -104,63 +87,59 @@ public: * type as this. */ void applyTo(Document& doc) const; - - /** - * Add a field update to this document update. - * @return A reference to this. - */ - DocumentUpdate& addUpdate(const FieldUpdate& update); - /** - * Add a fieldpath update to this document update. - * @return A reference to this. - */ + DocumentUpdate& addUpdate(const FieldUpdate& update); DocumentUpdate& addFieldPathUpdate(const FieldPathUpdate::CP& update); /** @return The list of updates. */ - const FieldUpdateV & getUpdates() const { return _updates; } + const FieldUpdateV & getUpdates() const; /** @return The list of fieldpath updates. */ - const FieldPathUpdateV & getFieldPathUpdates() const { return _fieldPathUpdates; } + const FieldPathUpdateV & getFieldPathUpdates() const; + + void eagerDeserialize() const; /** @return The type of document this update is for. */ const DocumentType& getType() const; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void serializeHEAD(vespalib::nbostream &stream) const; - void printXml(XmlOutputStream&) const override; /** * Sets whether this update should create the document it updates if that document does not exist. * In this case an empty document is created before the update is applied. */ - void setCreateIfNonExistent(bool value) { - _createIfNonExistent = value; - } + void setCreateIfNonExistent(bool value); /** * Gets whether this update should create the document it updates if that document does not exist. */ - bool getCreateIfNonExistent() const { - return _createIfNonExistent; - } + bool getCreateIfNonExistent() const; int serializeFlags(int size_) const; - int16_t getVersion() const { return _version; } private: - DocumentId _documentId; // The ID of the document to update. - const DataType *_type; // The type of document this update is for. - FieldUpdateV _updates; // The list of field updates. - FieldPathUpdateV _fieldPathUpdates; - int16_t _version; // Serialization version - bool _createIfNonExistent; + DocumentId _documentId; // The ID of the document to update. + const DataType *_type; // The type of document this update is for. + const DocumentTypeRepo *_repo; + vespalib::nbostream _backing; + FieldUpdateV _updates; // The list of field updates. + FieldPathUpdateV _fieldPathUpdates; + bool _createIfNonExistent; + bool _needHardReserialize; int deserializeFlags(int sizeAndFlags); - void deserialize42(const DocumentTypeRepo&, ByteBuffer&); - void deserializeHEAD(const DocumentTypeRepo&, ByteBuffer&); + void init42(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); + void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + void deserialize42(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + void deserializeBody(const DocumentTypeRepo &repo, vespalib::nbostream &stream); + void lazyDeserialize(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + void ensureDeserialized() const; + void serializeHeader(); + void reserialize(); + friend VespaDocumentSerializer; }; -} // document +} + diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp index 239ffff2fef..e7d5824b6e0 100644 --- a/document/src/vespa/document/update/fieldpathupdate.cpp +++ b/document/src/vespa/document/update/fieldpathupdate.cpp @@ -5,7 +5,7 @@ #include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/select/parser.h> #include <vespa/document/util/serializableexceptions.h> -#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/objects/nbostream.h> #include <ostream> #include <vespa/log/log.h> @@ -106,29 +106,29 @@ FieldPathUpdate::getResultingDataType(const FieldPath & path) const return path.back().getDataType(); } -vespalib::string -FieldPathUpdate::getString(ByteBuffer& buffer) +vespalib::stringref +FieldPathUpdate::getString(nbostream & stream) { - int32_t length = 0; - buffer.getIntNetwork(length); - vespalib::string s(buffer.getBufferAtPos()); - buffer.incPos(length); + uint32_t sz(0); + stream >> sz; + + vespalib::stringref s(stream.peek(), sz - 1); + stream.adjustReadPos(sz); return s; } void -FieldPathUpdate::deserialize(const DocumentTypeRepo&, const DataType&, ByteBuffer& buffer, uint16_t) +FieldPathUpdate::deserialize(const DocumentTypeRepo&, const DataType&, nbostream & stream) { - _originalFieldPath = getString(buffer); - _originalWhereClause = getString(buffer); + _originalFieldPath = getString(stream); + _originalWhereClause = getString(stream); } std::unique_ptr<FieldPathUpdate> -FieldPathUpdate::createInstance(const DocumentTypeRepo& repo, const DataType &type, - ByteBuffer& buffer, int serializationVersion) +FieldPathUpdate::createInstance(const DocumentTypeRepo& repo, const DataType &type, nbostream& stream) { unsigned char updateType = 0; - buffer.getByte(updateType); + stream >> updateType; std::unique_ptr<FieldPathUpdate> update; switch (updateType) { @@ -144,7 +144,7 @@ FieldPathUpdate::createInstance(const DocumentTypeRepo& repo, const DataType &ty default: throw DeserializeException(make_string("Unknown fieldpath update type: %d", updateType), VESPA_STRLOC); } - update->deserialize(repo, type, buffer, serializationVersion); + update->deserialize(repo, type, stream); return update; } diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index 76d529fec68..3e19420d0d0 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -23,13 +23,14 @@ class FieldPathUpdate : public vespalib::Cloneable, public vespalib::Identifiable { protected: + using nbostream = vespalib::nbostream; using stringref = vespalib::stringref; /** To be used for deserialization */ FieldPathUpdate(); FieldPathUpdate(const FieldPathUpdate &); FieldPathUpdate & operator =(const FieldPathUpdate &); - static vespalib::string getString(ByteBuffer& buffer); + static stringref getString(nbostream & stream); public: using SP = std::shared_ptr<FieldPathUpdate>; using CP = vespalib::CloneablePtr<FieldPathUpdate>; @@ -69,26 +70,14 @@ public: virtual uint8_t getSerializedType() const = 0; /** Deserializes and creates a new FieldPathUpdate instance. - * Requires type id to be not yet read. + * Requires type id to be not yet consumed. */ - static std::unique_ptr<FieldPathUpdate> createInstance( - const DocumentTypeRepo& repo, - const DataType &type, ByteBuffer& buffer, - int serializationVersion); + static std::unique_ptr<FieldPathUpdate> createInstance(const DocumentTypeRepo& repo, const DataType &type, nbostream & stream); protected: FieldPathUpdate(stringref fieldPath, stringref whereClause = stringref()); - /** - * Deserializes the given byte buffer into an instance of a FieldPathUpdate - * object. - * - * @param type A data type that describes the content of the buffer. - * @param buffer The byte buffer that contains the serialized object. - * @param version The serialization version of the object to deserialize. - */ - virtual void deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version); + virtual void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream); /** @return the datatype of the last path element in the field path */ const DataType& getResultingDataType(const FieldPath & path) const; diff --git a/document/src/vespa/document/update/fieldpathupdates.h b/document/src/vespa/document/update/fieldpathupdates.h index db2da82f705..fb2cf1dffe2 100644 --- a/document/src/vespa/document/update/fieldpathupdates.h +++ b/document/src/vespa/document/update/fieldpathupdates.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/update/addfieldpathupdate.h> -#include <vespa/document/update/assignfieldpathupdate.h> -#include <vespa/document/update/removefieldpathupdate.h> +#include "addfieldpathupdate.h" +#include "assignfieldpathupdate.h" +#include "removefieldpathupdate.h" diff --git a/document/src/vespa/document/update/fieldupdate.cpp b/document/src/vespa/document/update/fieldupdate.cpp index 4644ae1f340..d36a134ecaa 100644 --- a/document/src/vespa/document/update/fieldupdate.cpp +++ b/document/src/vespa/document/update/fieldupdate.cpp @@ -4,11 +4,11 @@ #include <vespa/document/base/exceptions.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/datatype/documenttype.h> -#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/objects/nbostream.h> namespace document { -typedef std::vector<ValueUpdate::CP> ValueUpdateList; +using vespalib::nbostream; FieldUpdate::FieldUpdate(const Field& field) : Printable(), @@ -19,24 +19,24 @@ FieldUpdate::FieldUpdate(const Field& field) namespace { -int readInt(ByteBuffer & buffer) { +int readInt(nbostream & stream) { int tmp; - buffer.getIntNetwork(tmp); + stream >> tmp; return tmp; } } -FieldUpdate::FieldUpdate(const DocumentTypeRepo& repo, const DataType & type, ByteBuffer& buffer, int16_t version) +FieldUpdate::FieldUpdate(const DocumentTypeRepo& repo, const DataType & type, nbostream & stream) : Printable(), - _field(type.getField(readInt(buffer))), + _field(type.getField(readInt(stream))), _updates() { - int numUpdates = readInt(buffer); + int numUpdates = readInt(stream); _updates.reserve(numUpdates); const DataType& dataType = _field.getDataType(); for(int i(0); i < numUpdates; i++) { - _updates.emplace_back(ValueUpdate::createInstance(repo, dataType, buffer, version).release()); + _updates.emplace_back(ValueUpdate::createInstance(repo, dataType, stream).release()); } } @@ -104,20 +104,17 @@ FieldUpdate::print(std::ostream& out, bool verbose, const std::string& indent) c // Deserialize this field update from the given buffer. void -FieldUpdate::deserialize(const DocumentTypeRepo& repo, const DocumentType& docType, - ByteBuffer& buffer, int16_t version) +FieldUpdate::deserialize(const DocumentTypeRepo& repo, const DocumentType& docType, nbostream& stream) { - int fieldId; - buffer.getIntNetwork(fieldId); + int fieldId = readInt(stream); _field = docType.getField(fieldId); const DataType& dataType = _field.getDataType(); - int numUpdates = 0; - buffer.getIntNetwork(numUpdates); + int numUpdates = readInt(stream); _updates.clear(); _updates.resize(numUpdates); for(int i = 0; i < numUpdates; i++) { - _updates[i].reset(ValueUpdate::createInstance(repo, dataType, buffer, version).release()); + _updates[i].reset(ValueUpdate::createInstance(repo, dataType, stream).release()); } } diff --git a/document/src/vespa/document/update/fieldupdate.h b/document/src/vespa/document/update/fieldupdate.h index fa5bac1cca9..6bd46c3a172 100644 --- a/document/src/vespa/document/update/fieldupdate.h +++ b/document/src/vespa/document/update/fieldupdate.h @@ -26,6 +26,7 @@ class FieldUpdate : public vespalib::Identifiable, { Field _field; std::vector<ValueUpdate::CP> _updates; + using nbostream = vespalib::nbostream; public: typedef vespalib::CloneablePtr<FieldUpdate> CP; @@ -39,14 +40,12 @@ public: /** * This is a convenience function to construct a field update directly from - * a byte buffer by deserializing all its content from the buffer. + * a stream by deserializing all its content from the stream. * - * @param type A document type that describes the buffer content. - * @param buffer A byte buffer that contains a serialized field update. - * @param serializationVersion The serialization version the update was serialized with. + * @param type A document type that describes the stream content. + * @param stream A stream that contains a serialized field update. */ - FieldUpdate(const DocumentTypeRepo& repo, const DataType & type, - ByteBuffer& buffer, int16_t version); + FieldUpdate(const DocumentTypeRepo& repo, const DataType & type, nbostream & stream); bool operator==(const FieldUpdate&) const; bool operator!=(const FieldUpdate & rhs) const { return ! (*this == rhs); } @@ -71,30 +70,18 @@ public: const std::vector<ValueUpdate::CP>& getUpdates() const { return _updates; } const Field& getField() const { return _field; } - - /** - * Applies this update object to the given {@link Document} object. - * - * @param doc The document to apply this update to. - */ void applyTo(Document& doc) const; - - // Printable implementation void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - // XmlSerializable implementation void printXml(XmlOutputStream&) const override; /** - * Deserializes the given byte buffer into an instance of an update object. + * Deserializes the given stream into an instance of an update object. * Not a Deserializable, as document type is needed as extra information. * - * @param type A document type that describes the buffer content. - * @param buffer The byte buffer that contains the serialized update object. - * @param serializationVersion The serialization version the update was serialized with. + * @param type A document type that describes the stream content. + * @param buffer The stream that contains the serialized update object. */ - void deserialize(const DocumentTypeRepo& repo, const DocumentType& type, - ByteBuffer& buffer, int16_t serializationVersion); + void deserialize(const DocumentTypeRepo& repo, const DocumentType& type, nbostream& stream); }; diff --git a/document/src/vespa/document/update/mapvalueupdate.cpp b/document/src/vespa/document/update/mapvalueupdate.cpp index 568b661e03d..ff1063a3dda 100644 --- a/document/src/vespa/document/update/mapvalueupdate.cpp +++ b/document/src/vespa/document/update/mapvalueupdate.cpp @@ -5,7 +5,6 @@ #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/document/util/serializableexceptions.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/util/xmlstream.h> #include <vespa/vespalib/objects/nbostream.h> @@ -128,18 +127,16 @@ MapValueUpdate::printXml(XmlOutputStream& xos) const // Deserialize this update from the given buffer. void -MapValueUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) +MapValueUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) { - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - VespaDocumentDeserializer deserializer(repo, stream, version); + VespaDocumentDeserializer deserializer(repo, stream, Document::getNewestSerializationVersion()); switch(type.getClass().id()) { case ArrayDataType::classId: { _key.reset(new IntFieldValue); deserializer.read(*_key); - buffer.incPos(buffer.getRemaining() - stream.size()); const ArrayDataType& arrayType = static_cast<const ArrayDataType&>(type); - _update.reset(ValueUpdate::createInstance(repo, arrayType.getNestedType(), buffer, version).release()); + _update.reset(ValueUpdate::createInstance(repo, arrayType.getNestedType(), stream).release()); break; } case WeightedSetDataType::classId: @@ -147,8 +144,7 @@ MapValueUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, const WeightedSetDataType& wset(static_cast<const WeightedSetDataType&>(type)); _key.reset(wset.getNestedType().createFieldValue().release()); deserializer.read(*_key); - buffer.incPos(buffer.getRemaining() - stream.size()); - _update.reset(ValueUpdate::createInstance(repo, *DataType::INT, buffer, version).release()); + _update.reset(ValueUpdate::createInstance(repo, *DataType::INT, stream).release()); break; } default: diff --git a/document/src/vespa/document/update/mapvalueupdate.h b/document/src/vespa/document/update/mapvalueupdate.h index 3cfab4fff69..4e407097405 100644 --- a/document/src/vespa/document/update/mapvalueupdate.h +++ b/document/src/vespa/document/update/mapvalueupdate.h @@ -18,8 +18,6 @@ namespace document { class MapValueUpdate : public ValueUpdate { FieldValue::CP _key; // The field value this update is mapping to. - // This is shared pointer to be able to lookup key - // in weighted set map. ValueUpdate::CP _update; //The update to apply to the value member of this. // Used by ValueUpdate's static factory function @@ -47,11 +45,9 @@ public: bool operator==(const ValueUpdate& other) const override; - /** @return The key of the field value to update. */ const FieldValue& getKey() const { return *_key; } FieldValue& getKey() { return *_key; } - /** @return The update to apply to the field value of this. */ const ValueUpdate& getUpdate() const { return *_update; } ValueUpdate& getUpdate() { return *_update; } @@ -77,18 +73,16 @@ public: return *this; } - // ValueUpdate implementation void checkCompatibility(const Field& field) const override; bool applyTo(FieldValue& value) const override; void printXml(XmlOutputStream& xos) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) override; + void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream& buffer) override; MapValueUpdate* clone() const override { return new MapValueUpdate(*this); } DECLARE_IDENTIFIABLE(MapValueUpdate); }; -} // document +} diff --git a/document/src/vespa/document/update/removefieldpathupdate.cpp b/document/src/vespa/document/update/removefieldpathupdate.cpp index c0ae2f869e4..57832055001 100644 --- a/document/src/vespa/document/update/removefieldpathupdate.cpp +++ b/document/src/vespa/document/update/removefieldpathupdate.cpp @@ -36,10 +36,9 @@ RemoveFieldPathUpdate::print(std::ostream& out, bool verbose, const std::string& } void -RemoveFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) +RemoveFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) { - FieldPathUpdate::deserialize(repo, type, buffer, version); + FieldPathUpdate::deserialize(repo, type, stream); } namespace { diff --git a/document/src/vespa/document/update/removefieldpathupdate.h b/document/src/vespa/document/update/removefieldpathupdate.h index db7e234bfc8..1329361621e 100644 --- a/document/src/vespa/document/update/removefieldpathupdate.h +++ b/document/src/vespa/document/update/removefieldpathupdate.h @@ -23,10 +23,10 @@ public: private: uint8_t getSerializedType() const override { return RemoveMagic; } - void deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) override; + void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & buffer) override; std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document &, const DocumentTypeRepo &) const override; }; -} // ns document +} + diff --git a/document/src/vespa/document/update/removevalueupdate.cpp b/document/src/vespa/document/update/removevalueupdate.cpp index c21e7c93abe..28c69652a0e 100644 --- a/document/src/vespa/document/update/removevalueupdate.cpp +++ b/document/src/vespa/document/update/removevalueupdate.cpp @@ -7,7 +7,6 @@ #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/util/xmlstream.h> using vespalib::IllegalArgumentException; @@ -85,7 +84,7 @@ RemoveValueUpdate::print(std::ostream& out, bool, const std::string&) const // Deserialize this update from the given buffer. void -RemoveValueUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) +RemoveValueUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) { switch(type.getClass().id()) { case ArrayDataType::classId: @@ -93,10 +92,8 @@ RemoveValueUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& typ { const CollectionDataType& c(static_cast<const CollectionDataType&>(type)); _key.reset(c.getNestedType().createFieldValue().release()); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - VespaDocumentDeserializer deserializer(repo, stream, version); + VespaDocumentDeserializer deserializer(repo, stream, Document::getNewestSerializationVersion()); deserializer.read(*_key); - buffer.incPos(buffer.getRemaining() - stream.size()); break; } default: diff --git a/document/src/vespa/document/update/removevalueupdate.h b/document/src/vespa/document/update/removevalueupdate.h index 6912a6e0bf3..c674ebf430c 100644 --- a/document/src/vespa/document/update/removevalueupdate.h +++ b/document/src/vespa/document/update/removevalueupdate.h @@ -46,18 +46,16 @@ public: return *this; } - // ValueUpdate implementation void checkCompatibility(const Field& field) const override; bool applyTo(FieldValue& value) const override; void printXml(XmlOutputStream& xos) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void deserialize(const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) override; + void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream& buffer) override; RemoveValueUpdate* clone() const override { return new RemoveValueUpdate(*this); } DECLARE_IDENTIFIABLE(RemoveValueUpdate); }; -} // document +} diff --git a/document/src/vespa/document/update/updates.h b/document/src/vespa/document/update/updates.h index 92cb51699cd..50a6e318b6a 100644 --- a/document/src/vespa/document/update/updates.h +++ b/document/src/vespa/document/update/updates.h @@ -2,13 +2,12 @@ #pragma once -#include <vespa/document/update/documentupdate.h> -#include <vespa/document/update/fieldupdate.h> - -#include <vespa/document/update/addvalueupdate.h> -#include <vespa/document/update/arithmeticvalueupdate.h> -#include <vespa/document/update/assignvalueupdate.h> -#include <vespa/document/update/clearvalueupdate.h> -#include <vespa/document/update/mapvalueupdate.h> -#include <vespa/document/update/removevalueupdate.h> +#include "documentupdate.h" +#include "fieldupdate.h" +#include "addvalueupdate.h" +#include "arithmeticvalueupdate.h" +#include "assignvalueupdate.h" +#include "clearvalueupdate.h" +#include "mapvalueupdate.h" +#include "removevalueupdate.h" diff --git a/document/src/vespa/document/update/valueupdate.cpp b/document/src/vespa/document/update/valueupdate.cpp index 103d9341f0e..6b893a80fd0 100644 --- a/document/src/vespa/document/update/valueupdate.cpp +++ b/document/src/vespa/document/update/valueupdate.cpp @@ -2,31 +2,29 @@ #include "valueupdate.h" #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/objects/nbostream.h> #include <stdexcept> namespace document { IMPLEMENT_IDENTIFIABLE_ABSTRACT(ValueUpdate, Identifiable); -// Create a value update from a byte buffer. std::unique_ptr<ValueUpdate> -ValueUpdate::createInstance(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, int serializationVersion) +ValueUpdate::createInstance(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) { - ValueUpdate* update(NULL); int32_t classId = 0; - buffer.getIntNetwork(classId); + stream >> classId; const Identifiable::RuntimeClass * rtc(Identifiable::classFromId(classId)); - if (rtc != NULL) { - update = static_cast<ValueUpdate*>(Identifiable::classFromId(classId)->create()); - /// \todo TODO (was warning): Updates are not versioned in serialization format. Will not work with altering it. - update->deserialize(repo, type, buffer, serializationVersion); + if (rtc != nullptr) { + std::unique_ptr<ValueUpdate> update(static_cast<ValueUpdate*>(rtc->create())); + /// \todo TODO (was warning): Updates are not versioned in serialization format. Will not work without altering it. + /// Should also use the serializer, not this deserialize into self. + update->deserialize(repo, type, stream); + return update; } else { throw std::runtime_error(vespalib::make_string("Could not find a class for classId %d(%x)", classId, classId)); } - - return std::unique_ptr<ValueUpdate>(update); } } diff --git a/document/src/vespa/document/update/valueupdate.h b/document/src/vespa/document/update/valueupdate.h index ffa40b1c191..963e1ad1d96 100644 --- a/document/src/vespa/document/update/valueupdate.h +++ b/document/src/vespa/document/update/valueupdate.h @@ -24,7 +24,6 @@ namespace document { -class ByteBuffer; class DocumentTypeRepo; class Field; class FieldValue; @@ -34,18 +33,18 @@ class ValueUpdate : public vespalib::Identifiable, public vespalib::Cloneable, public XmlSerializable { +protected: + using nbostream = vespalib::nbostream; public: - typedef vespalib::CloneablePtr<ValueUpdate> CP; + using CP = vespalib::CloneablePtr<ValueUpdate>; /** - * Create a value update object from the given byte buffer. + * Create a value update object from the given stream. * * @param type A data type that describes the content of the buffer. - * @param buffer The byte buffer that containes the serialized update. + * @param buffer The stream that containes the serialized update. */ - static std::unique_ptr<ValueUpdate> createInstance( - const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, int serializationVersion); + static std::unique_ptr<ValueUpdate> createInstance(const DocumentTypeRepo& repo, const DataType& type, nbostream & buffer); /** Define all types of value updates. */ enum ValueUpdateType { @@ -69,8 +68,7 @@ public: * Recursively checks the compatibility of this value update as * applied to the given document field. * - * @throws IllegalArgumentException Thrown if this value update - * is not compatible. + * @throws IllegalArgumentException Thrown if this value update is not compatible. */ virtual void checkCompatibility(const Field& field) const = 0; @@ -84,15 +82,12 @@ public: ValueUpdate* clone() const override = 0; /** - * Deserializes the given byte buffer into an instance of an update object. + * Deserializes the given stream into an instance of an update object. * - * @param type A data type that describes the content of the buffer. - * @param buffer The byte buffer that contains the serialized update object. - * @param version The serialization version of the object to deserialize. + * @param type A data type that describes the content of the stream. + * @param buffer The stream that contains the serialized update object. */ - virtual void deserialize(const DocumentTypeRepo& repo, - const DataType& type, - ByteBuffer& buffer, uint16_t version) = 0; + virtual void deserialize(const DocumentTypeRepo& repo, const DataType& type, nbostream & stream) = 0; /** @return The operation type. */ ValueUpdateType getType() const { @@ -105,8 +100,7 @@ public: virtual void accept(UpdateVisitor &visitor) const = 0; DECLARE_IDENTIFIABLE_ABSTRACT(ValueUpdate); - }; -} // document +} diff --git a/documentapi/src/tests/messagebus/messagebus_test.cpp b/documentapi/src/tests/messagebus/messagebus_test.cpp index cd2820d8c05..dff3f3f1bac 100644 --- a/documentapi/src/tests/messagebus/messagebus_test.cpp +++ b/documentapi/src/tests/messagebus/messagebus_test.cpp @@ -59,7 +59,7 @@ void Test::testMessage() { // Test one update. UpdateDocumentMessage upd1( document::DocumentUpdate::SP( - new document::DocumentUpdate(*testdoc_type, + new document::DocumentUpdate(*_repo, *testdoc_type, document::DocumentId(document::DocIdString( "testdoc", "testme1"))))); @@ -80,7 +80,7 @@ void Test::testMessage() { // Compare to another. UpdateDocumentMessage upd2( document::DocumentUpdate::SP( - new document::DocumentUpdate(*testdoc_type, + new document::DocumentUpdate(*_repo, *testdoc_type, document::DocumentId(document::DocIdString( "testdoc", "testme2"))))); EXPECT_TRUE(!(upd1.getDocumentUpdate().getId() == upd2.getDocumentUpdate().getId())); diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp index 48728ff6057..6705277d5c3 100644 --- a/documentapi/src/tests/messages/messages50test.cpp +++ b/documentapi/src/tests/messages/messages50test.cpp @@ -672,8 +672,7 @@ Messages50Test::testUpdateDocumentMessage() { const DocumentTypeRepo &repo = getTypeRepo(); const document::DocumentType &docType = *repo.getDocumentType("testdoc"); - document::DocumentUpdate::SP - upd(new document::DocumentUpdate(docType, document::DocumentId("doc:scheme:"))); + auto upd(std::make_shared<document::DocumentUpdate>(repo, docType, document::DocumentId("doc:scheme:"))); upd->addFieldPathUpdate(document::FieldPathUpdate::CP( new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); UpdateDocumentMessage msg(upd); @@ -682,7 +681,7 @@ Messages50Test::testUpdateDocumentMessage() EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 89u, serialize("UpdateDocumentMessage", msg)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { mbus::Routable::UP obj = deserialize("UpdateDocumentMessage", DocumentProtocol::MESSAGE_UPDATEDOCUMENT, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { + if (EXPECT_TRUE(obj.get() != nullptr)) { UpdateDocumentMessage &ref = static_cast<UpdateDocumentMessage&>(*obj); EXPECT_EQUAL(*upd, ref.getDocumentUpdate()); EXPECT_EQUAL(666u, ref.getOldTimestamp()); @@ -703,21 +702,21 @@ Messages50Test::testBatchDocumentUpdateMessage() { document::DocumentUpdate::SP upd; - upd.reset(new document::DocumentUpdate(docType, document::DocumentId("userdoc:footype:1234:foo"))); + upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("userdoc:footype:1234:foo"))); upd->addFieldPathUpdate(document::FieldPathUpdate::CP( new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); msg.addUpdate(upd); } { document::DocumentUpdate::SP upd; - upd.reset(new document::DocumentUpdate(docType, document::DocumentId("orderdoc(32,17):footype:1234:123456789:foo"))); + upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("orderdoc(32,17):footype:1234:123456789:foo"))); upd->addFieldPathUpdate(document::FieldPathUpdate::CP( new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); msg.addUpdate(upd); } try { document::DocumentUpdate::SP upd; - upd.reset(new document::DocumentUpdate(docType, document::DocumentId("userdoc:footype:5678:foo"))); + upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("userdoc:footype:5678:foo"))); upd->addFieldPathUpdate(document::FieldPathUpdate::CP( new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); msg.addUpdate(upd); @@ -726,7 +725,7 @@ Messages50Test::testBatchDocumentUpdateMessage() } try { document::DocumentUpdate::SP upd; - upd.reset(new document::DocumentUpdate(docType, document::DocumentId("groupdoc:footype:hable:foo"))); + upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("groupdoc:footype:hable:foo"))); upd->addFieldPathUpdate(document::FieldPathUpdate::CP( new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); msg.addUpdate(upd); diff --git a/documentapi/src/tests/messages/messages52test.cpp b/documentapi/src/tests/messages/messages52test.cpp index eaf8bbec8c4..8f5d7381500 100644 --- a/documentapi/src/tests/messages/messages52test.cpp +++ b/documentapi/src/tests/messages/messages52test.cpp @@ -91,7 +91,7 @@ Messages52Test::testUpdateDocumentMessage() const DocumentTypeRepo & repo = getTypeRepo(); const document::DocumentType & docType = *repo.getDocumentType("testdoc"); - auto docUpdate = std::make_shared<document::DocumentUpdate>(docType, document::DocumentId("doc:scheme:")); + auto docUpdate = std::make_shared<document::DocumentUpdate>(repo, docType, document::DocumentId("doc:scheme:")); docUpdate->addFieldPathUpdate(document::FieldPathUpdate::CP( new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp index 7852136dfc7..3e804c30415 100644 --- a/documentapi/src/tests/policies/policies_test.cpp +++ b/documentapi/src/tests/policies/policies_test.cpp @@ -621,7 +621,7 @@ Test::testDocumentRouteSelector() EXPECT_TRUE(frame.testSelect(StringList().add("foo").add("bar"))); frame.setMessage(make_unique<UpdateDocumentMessage>( - make_shared<DocumentUpdate>(*_docType, DocumentId("doc:scheme:")))); + make_shared<DocumentUpdate>(*_repo, *_docType, DocumentId("doc:scheme:")))); EXPECT_TRUE(frame.testSelect(StringList().add("foo"))); put = make_unique<PutDocumentMessage>(make_shared<Document>(*_docType, DocumentId("doc:scheme:"))); @@ -650,7 +650,7 @@ Test::testDocumentRouteSelectorIgnore() EXPECT_EQUAL(0u, reply->getNumErrors()); frame.setMessage(make_unique<UpdateDocumentMessage>( - make_shared<DocumentUpdate>(*_docType, DocumentId("doc:scheme:")))); + make_shared<DocumentUpdate>(*_repo, *_docType, DocumentId("doc:scheme:")))); EXPECT_TRUE(frame.testSelect(StringList().add("docproc/cluster.foo"))); } diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsStrategyFactory.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsStrategyFactory.java index 6523a0c138f..1459024767d 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsStrategyFactory.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsStrategyFactory.java @@ -53,7 +53,7 @@ public class JaxRsStrategyFactory { this.scheme = scheme; } - public <T> JaxRsStrategy<T> apiWithRetries(final Class<T> apiClass, final String pathPrefix) { + public <T> RetryingJaxRsStrategy<T> apiWithRetries(final Class<T> apiClass, final String pathPrefix) { Objects.requireNonNull(apiClass, "apiClass argument may not be null"); Objects.requireNonNull(pathPrefix, "pathPrefix argument may not be null"); return new RetryingJaxRsStrategy<T>(hostNames, port, jaxRsClientFactory, apiClass, pathPrefix, scheme); diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java index 73320a4c72d..65b302ef4ff 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java @@ -21,7 +21,6 @@ import java.util.logging.Logger; */ public class RetryingJaxRsStrategy<T> implements JaxRsStrategy<T> { private static final Logger logger = Logger.getLogger(RetryingJaxRsStrategy.class.getName()); - private static final int NUM_LOOP_ATTEMPTS = 2; private final List<HostName> hostNames; private final int port; @@ -30,6 +29,8 @@ public class RetryingJaxRsStrategy<T> implements JaxRsStrategy<T> { private String pathPrefix; private final String scheme; + private int maxIterations = 2; + public RetryingJaxRsStrategy( final Set<HostName> hostNames, final int port, @@ -52,11 +53,21 @@ public class RetryingJaxRsStrategy<T> implements JaxRsStrategy<T> { this.scheme = scheme; } + /** + * The the max number of times the hostnames should be iterated over, before giving up. + * + * <p>By default, maxIterations is 2. + */ + public RetryingJaxRsStrategy<T> setMaxIterations(int maxIterations) { + this.maxIterations = maxIterations; + return this; + } + @Override public <R> R apply(final Function<T, R> function) throws IOException { ProcessingException sampleException = null; - for (int i = 0; i < NUM_LOOP_ATTEMPTS; ++i) { + for (int i = 0; i < maxIterations; ++i) { for (final HostName hostName : hostNames) { final T jaxRsClient = jaxRsClientFactory.createClient(apiClass, hostName, port, pathPrefix, scheme); try { @@ -72,7 +83,7 @@ public class RetryingJaxRsStrategy<T> implements JaxRsStrategy<T> { final String message = String.format( "Giving up invoking REST API after %d tries against hosts %s.%s", - NUM_LOOP_ATTEMPTS, + maxIterations, hostNames, sampleException == null ? "" : ", sample error: " + sampleException.getMessage()); diff --git a/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategyTest.java b/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategyTest.java index 10dde1ff820..e31920febd6 100644 --- a/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategyTest.java +++ b/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategyTest.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.defaults.Defaults; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; +import org.mockito.stubbing.OngoingStubbing; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -47,7 +48,7 @@ public class RetryingJaxRsStrategyTest { private final JaxRsClientFactory jaxRsClientFactory = mock(JaxRsClientFactory.class); private final TestJaxRsApi mockApi = mock(TestJaxRsApi.class); - private final JaxRsStrategy<TestJaxRsApi> jaxRsStrategy = new RetryingJaxRsStrategy<>( + private final RetryingJaxRsStrategy<TestJaxRsApi> jaxRsStrategy = new RetryingJaxRsStrategy<>( SERVER_HOSTS, REST_PORT, jaxRsClientFactory, TestJaxRsApi.class, API_PATH, "http"); @Before @@ -111,14 +112,24 @@ public class RetryingJaxRsStrategyTest { } @Test - public void testRetryGivesUpAfterTwoLoopsOverAvailableServers() throws Exception { - when(mockApi.doSomething()) - .thenThrow(new ProcessingException("Fake timeout 1 induced by test")) - .thenThrow(new ProcessingException("Fake timeout 2 induced by test")) - .thenThrow(new ProcessingException("Fake timeout 3 induced by test")) - .thenThrow(new ProcessingException("Fake timeout 4 induced by test")) - .thenThrow(new ProcessingException("Fake timeout 5 induced by test")) - .thenThrow(new ProcessingException("Fake timeout 6 induced by test")); + public void testRetryGivesUpAfterOneLoopOverAvailableServers() { + jaxRsStrategy.setMaxIterations(1); + testRetryGivesUpAfterXIterations(1); + } + + @Test + public void testRetryGivesUpAfterTwoLoopsOverAvailableServers() { + testRetryGivesUpAfterXIterations(2); + } + + private void testRetryGivesUpAfterXIterations(int iterations) { + OngoingStubbing<String> stub = when(mockApi.doSomething()); + for (int i = 0; i < iterations; ++i) { + stub = stub + .thenThrow(new ProcessingException("Fake timeout 1 iteration " + i)) + .thenThrow(new ProcessingException("Fake timeout 2 iteration " + i)) + .thenThrow(new ProcessingException("Fake timeout 3 iteration " + i)); + } try { jaxRsStrategy.apply(TestJaxRsApi::doSomething); @@ -127,7 +138,7 @@ public class RetryingJaxRsStrategyTest { // As expected. } - verify(mockApi, times(6)).doSomething(); + verify(mockApi, times(iterations * 3)).doSomething(); verifyAllServersContacted(jaxRsClientFactory); } diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/SecurityHeadersResponseFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/SecurityHeadersResponseFilter.java index f50e7454f19..09d02d66b1f 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/SecurityHeadersResponseFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/SecurityHeadersResponseFilter.java @@ -17,5 +17,6 @@ public class SecurityHeadersResponseFilter implements SecurityResponseFilter { response.setHeader("Cache-control", "no-store"); response.setHeader("Pragma", "no-cache"); response.setHeader("Strict-Transport-Security", "max-age=31536000; includeSubDomains"); + response.setHeader("X-Content-Type-Options", "nosniff"); } } diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index 55da222994f..246265203cb 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -147,13 +147,6 @@ <version>${project.version}</version> <scope>compile</scope> </dependency> - <dependency> - <!-- This seems odd. Used for export-package parsing. Lazy stuff. Should be separated out. --> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> </dependencies> <build> <plugins> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index f2e9c1a6e53..8da1a4fad99 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -1,11 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.core; -import com.yahoo.container.plugin.bundle.TransformExportPackages; -import com.yahoo.container.plugin.osgi.ExportPackages.Export; import org.apache.felix.framework.util.Util; import org.osgi.framework.Constants; -import scala.collection.immutable.List; import java.io.File; import java.io.FileInputStream; @@ -96,10 +93,4 @@ public class ExportPackages { return jar.getManifest().getMainAttributes().getValue(Constants.EXPORT_PACKAGE); } } - - private static String transformExports(List<Export> exports, String newVersion) { - return TransformExportPackages.toExportPackageProperty( - TransformExportPackages.removeUses( - TransformExportPackages.replaceVersions(exports, newVersion))); - } } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index 24ac0091e72..e6ce4eddb59 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -10,7 +10,7 @@ import com.yahoo.text.Utf8; import java.nio.ByteBuffer; /** - * <p>Includes functionality for determining the langCode from a sample or from the encoding. Currently only Chinese, + * Includes functionality for determining the langCode from a sample or from the encoding. Currently only Chinese, * Japanese and Korean are supported. There are two ways to guess a String's langCode, by encoding and by character * set. If the encoding is available this is a very good indication of the langCode. If the encoding is not available, * then the actual characters in the string can be used to make an educated guess at the String's langCode. Recall a @@ -18,7 +18,7 @@ import java.nio.ByteBuffer; * Unfortunately, its not 100% fool-proof. From what I've been able to determine, Korean characters do not overlap with * Japanese or Chinese characters, so their presence is a good indication of Korean. If a string contains phonetic * japanese, this is a good indication of Japanese. However, Japanese and Chinese characters occupy many of the same - * character blocks, so if there are no definitive signs of Japanese then it is assumed that the String is Chinese.</p> + * character blocks, so if there are no definitive signs of Japanese then it is assumed that the String is Chinese. * * @author Rich Pito */ @@ -137,7 +137,7 @@ public class SimpleDetector implements Detector { } } - private String guessEncoding(byte[] input) { + public String guessEncoding(byte[] input) { boolean isUtf8 = true; boolean hasHighs = false; scan: diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java index 20c41d657e1..122b9b6dff6 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java @@ -9,7 +9,7 @@ import java.util.ArrayList; import java.util.List; /** - * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a> + * @author Mathias Mølster Lidal */ public class SimpleToken implements Token { diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java index c9c6286336d..d7eb8a72ed8 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java @@ -6,7 +6,7 @@ import com.yahoo.language.process.TokenType; /** * @author arnej27959 */ -class SimpleTokenType { +public class SimpleTokenType { public static TokenType valueOf(int codePoint) { switch (Character.getType(codePoint)) { diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTransformer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTransformer.java index fce3344bfad..f0227a73946 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTransformer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTransformer.java @@ -11,7 +11,7 @@ import java.util.regex.Pattern; * Converts all accented characters into their de-accented counterparts followed by their combining diacritics, then * strips off the diacritics using a regex. * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class SimpleTransformer implements Transformer { diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index f3e3359fabe..108b94070bf 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -120,11 +120,13 @@ RPCNetwork::RPCNetwork(const RPCNetworkParams ¶ms) : _mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, *_slobrokCfgFactory)), _regAPI(std::make_unique<slobrok::api::RegisterAPI>(*_orb, *_slobrokCfgFactory)), _requestedPort(params.getListenPort()), - _executor(std::make_unique<vespalib::ThreadStackExecutor>(4,65536)), + _executor(std::make_unique<vespalib::ThreadStackExecutor>(params.getNumThreads(), 65536)), _sendV1(std::make_unique<RPCSendV1>()), _sendV2(std::make_unique<RPCSendV2>()), _sendAdapters(), - _compressionConfig(params.getCompressionConfig()) + _compressionConfig(params.getCompressionConfig()), + _allowDispatchForEncode(params.getDispatchOnEncode()), + _allowDispatchForDecode(params.getDispatchOnDecode()) { _transport->SetDirectWrite(false); _transport->SetMaxInputBufferSize(params.getMaxInputBufferSize()); @@ -224,11 +226,6 @@ RPCNetwork::start() return true; } -vespalib::Executor & -RPCNetwork::getExecutor() { - return *_executor; -} - bool RPCNetwork::waitUntilReady(double seconds) const { diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.h b/messagebus/src/vespa/messagebus/network/rpcnetwork.h index e29d01c8b04..9c6516eced7 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.h @@ -77,6 +77,9 @@ private: std::unique_ptr<RPCSendAdapter> _sendV2; SendAdapterMap _sendAdapters; CompressionConfig _compressionConfig; + bool _allowDispatchForEncode; + bool _allowDispatchForDecode; + /** * Resolves and assigns a service address for the given recipient using the @@ -135,7 +138,7 @@ public: /** * Destruct **/ - virtual ~RPCNetwork(); + ~RPCNetwork() override; /** * Obtain the owner of this network. This method may only be invoked after @@ -226,7 +229,10 @@ public: const slobrok::api::IMirrorAPI &getMirror() const override; CompressionConfig getCompressionConfig() { return _compressionConfig; } void invoke(FRT_RPCRequest *req); - vespalib::Executor & getExecutor(); + vespalib::Executor & getExecutor() const { return *_executor; } + bool allowDispatchForEncode() const { return _allowDispatchForEncode; } + bool allowDispatchForDecode() const { return _allowDispatchForDecode; } + }; } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp index 5e54de1bce6..b6f0231e619 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp @@ -10,6 +10,9 @@ RPCNetworkParams::RPCNetworkParams() : _listenPort(0), _maxInputBufferSize(256*1024), _maxOutputBufferSize(256*1024), + _numThreads(4), + _dispatchOnEncode(true), + _dispatchOnDecode(false), _connectionExpireSecs(600), _compressionConfig(CompressionConfig::LZ4, 6, 90, 1024) { } diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h index 0a4ed806c27..1dcc8178e68 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h @@ -19,6 +19,9 @@ private: int _listenPort; uint32_t _maxInputBufferSize; uint32_t _maxOutputBufferSize; + uint32_t _numThreads; + bool _dispatchOnEncode; + bool _dispatchOnDecode; double _connectionExpireSecs; CompressionConfig _compressionConfig; @@ -97,6 +100,19 @@ public: } /** + * Sets number of threads for the thread pool. + * + * @param numThreads number of threads for thread pool + * @return This, to allow chaining. + */ + RPCNetworkParams &setNumThreads(uint32_t numThreads) { + _numThreads = numThreads; + return *this; + } + + uint32_t getNumThreads() const { return _numThreads; } + + /** * Returns the number of seconds before an idle network connection expires. * * @return The number of seconds. @@ -165,6 +181,21 @@ public: return *this; } CompressionConfig getCompressionConfig() const { return _compressionConfig; } + + + RPCNetworkParams &setDispatchOnDecode(bool dispatchOnDecode) { + _dispatchOnDecode = dispatchOnDecode; + return *this; + } + + uint32_t getDispatchOnDecode() const { return _dispatchOnDecode; } + + RPCNetworkParams &setDispatchOnEncode(bool dispatchOnEncode) { + _dispatchOnEncode = dispatchOnEncode; + return *this; + } + + uint32_t getDispatchOnEncode() const { return _dispatchOnEncode; } }; } diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.cpp b/messagebus/src/vespa/messagebus/network/rpcsend.cpp index e23f4dc29d9..04cccd59903 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsend.cpp @@ -61,7 +61,7 @@ RPCSend::RPCSend() : _serverIdent("server") { } -RPCSend::~RPCSend() {} +RPCSend::~RPCSend() = default; void RPCSend::attach(RPCNetwork &net) @@ -221,7 +221,7 @@ void RPCSend::handleReply(Reply::UP reply) { const IProtocol * protocol = _net->getOwner().getProtocol(reply->getProtocol()); - if (!protocol || protocol->requireSequencing()) { + if (!protocol || protocol->requireSequencing() || !_net->allowDispatchForEncode()) { doHandleReply(protocol, std::move(reply)); } else { auto rejected = _net->getExecutor().execute(makeLambdaTask([this, protocol, reply = std::move(reply)]() mutable { @@ -256,22 +256,29 @@ void RPCSend::invoke(FRT_RPCRequest *req) { req->Detach(); - doRequest(req); -} - -void -RPCSend::doRequest(FRT_RPCRequest *req) -{ FRT_Values &args = *req->GetParams(); std::unique_ptr<Params> params = toParams(args); IProtocol * protocol = _net->getOwner().getProtocol(params->getProtocol()); if (protocol == nullptr) { replyError(req, params->getVersion(), params->getTraceLevel(), - Error(ErrorCode::UNKNOWN_PROTOCOL, - make_string("Protocol '%s' is not known by %s.", params->getProtocol().c_str(), _serverIdent.c_str()))); + Error(ErrorCode::UNKNOWN_PROTOCOL, make_string("Protocol '%s' is not known by %s.", + params->getProtocol().c_str(), _serverIdent.c_str()))); return; } + if (protocol->requireSequencing() || !_net->allowDispatchForDecode()) { + doRequest(req, protocol, std::move(params)); + } else { + auto rejected = _net->getExecutor().execute(makeLambdaTask([this, req, protocol, params = std::move(params)]() mutable { + doRequest(req, protocol, std::move(params)); + })); + assert (!rejected); + } +} + +void +RPCSend::doRequest(FRT_RPCRequest *req, const IProtocol * protocol, std::unique_ptr<Params> params) +{ Routable::UP routable = protocol->decode(params->getVersion(), params->getPayload()); req->DiscardBlobs(); if ( ! routable ) { diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.h b/messagebus/src/vespa/messagebus/network/rpcsend.h index 11a042b91c0..cfc1d72418a 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.h +++ b/messagebus/src/vespa/messagebus/network/rpcsend.h @@ -83,7 +83,7 @@ public: void invoke(FRT_RPCRequest *req); private: - void doRequest(FRT_RPCRequest *req); + void doRequest(FRT_RPCRequest *req, const IProtocol * protocol, std::unique_ptr<Params> params); void doRequestDone(FRT_RPCRequest *req); void doHandleReply(const IProtocol * protocol, std::unique_ptr<Reply> reply); void attach(RPCNetwork &net) final override; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java index 6092cc1f038..14fea240baa 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java @@ -13,7 +13,7 @@ import java.nio.file.Path; * Uses the Velocity engine to render a template, to and from both String and Path objects. * * @author hakonhall - * @author jvenstad + * @author jonmv */ public class Template { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 5024296b0a7..1383a0a8cf3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -661,4 +661,5 @@ public class NodeRepository extends AbstractComponent { private Mutex lock(Node node) { return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated(); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java index e78bcb6b5e8..49f8b704c5e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java @@ -97,6 +97,10 @@ class NodeIdentifier { } private String getHostFromVespaCertificate(List<SubjectAlternativeName> sans) { + // TODO Remove this branch once all BM nodes are gone + if (sans.stream().anyMatch(san -> san.getValue().endsWith("ostk.yahoo.cloud"))) { + return getHostFromCalypsoCertificate(sans); + } VespaUniqueInstanceId instanceId = VespaUniqueInstanceId.fromDottedString(getUniqueInstanceId(sans)); if (!zone.environment().value().equals(instanceId.environment())) throw new NodeIdentifierException("Invalid environment: " + instanceId.environment()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java index 9c441e82a84..445d18bed7c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java @@ -201,6 +201,25 @@ public class NodeIdentifierTest { assertEquals(CONTROLLER_IDENTITY, identity.getHostIdentityName()); } + @Test + public void accepts_openstack_bm_tenant_certificate() { + NodeRepositoryTester nodeRepositoryDummy = new NodeRepositoryTester(); + nodeRepositoryDummy.addNode(OPENSTACK_ID, HOSTNAME, INSTANCE_ID, NodeType.tenant); + nodeRepositoryDummy.setNodeState(HOSTNAME, Node.State.active); + Pkcs10Csr csr = Pkcs10CsrBuilder + .fromKeypair(new X500Principal("CN=" + TENANT_DOCKER_CONTAINER_IDENTITY), KEYPAIR, SHA256_WITH_RSA) + .build(); + X509Certificate certificate = X509CertificateBuilder + .fromCsr(csr, ATHENZ_YAHOO_CA_CERT.getSubjectX500Principal(), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), KEYPAIR.getPrivate(), SHA256_WITH_RSA, 1) + .addSubjectAlternativeName(OPENSTACK_ID + ".instanceid.athenz.ostk.yahoo.cloud") + .build(); + NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository()); + NodePrincipal identity = identifier.resolveNode(singletonList(certificate)); + assertTrue(identity.getHostname().isPresent()); + assertEquals(HOSTNAME, identity.getHostname().get()); + assertEquals(TENANT_DOCKER_CONTAINER_IDENTITY, identity.getHostIdentityName()); + } + private static Node createNode(String clusterId, int clusterIndex, String tenant, String application) { return Node .createDockerNode( diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java index c7aae6ea93d..5571eedeec6 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java @@ -41,7 +41,13 @@ public class RetryingClusterControllerClientFactory implements ClusterController Set<HostName> clusterControllerSet = clusterControllers.stream().collect(Collectors.toSet()); JaxRsStrategy<ClusterControllerJaxRsApi> jaxRsApi = new JaxRsStrategyFactory(clusterControllerSet, HARDCODED_CLUSTERCONTROLLER_PORT, jaxRsClientFactory, CLUSTERCONTROLLER_SCHEME) - .apiWithRetries(ClusterControllerJaxRsApi.class, CLUSTERCONTROLLER_API_PATH); + .apiWithRetries(ClusterControllerJaxRsApi.class, CLUSTERCONTROLLER_API_PATH) + // Use max iteration 1. The JaxRsStrategyFactory will try host 1, 2, then 3: + // - If host 1 responds, it will redirect to master if necessary. Otherwise + // - If host 2 responds, it will redirect to master if necessary. Otherwise + // - If host 3 responds, it may redirect to master if necessary (if they're up + // after all), but more likely there's no quorum and this will fail too. + .setMaxIterations(1); return new ClusterControllerClientImpl(jaxRsApi, clusterName); } diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index f9538a94c59..c1373a391f0 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -897,10 +897,8 @@ void ConformanceTest::testUpdate() { const document::DocumentType *docType( testDocMan.getTypeRepo().getDocumentType("testdoctype1")); - document::DocumentUpdate::SP - update(new DocumentUpdate(*docType, doc1->getId())); - std::shared_ptr<document::AssignValueUpdate> assignUpdate( - new document::AssignValueUpdate(document::IntFieldValue(42))); + document::DocumentUpdate::SP update(new DocumentUpdate(testDocMan.getTypeRepo(), *docType, doc1->getId())); + std::shared_ptr<document::AssignValueUpdate> assignUpdate(new document::AssignValueUpdate(document::IntFieldValue(42))); document::FieldUpdate fieldUpdate(docType->getField("headerval")); fieldUpdate.addUpdate(*assignUpdate); update->addUpdate(fieldUpdate); diff --git a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h index c7993503107..c432606f7f0 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h +++ b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h @@ -71,6 +71,11 @@ public: **/ virtual const vespalib::string & getName() const = 0; + vespalib::stringref getNamePrefix() const { + vespalib::stringref name = getName(); + return name.substr(0, name.find('.')); + } + /** * Returns the number of documents stored in this attribute vector. * diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index 17f1faffbba..8ffe4807427 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -1,8 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/config/helper/configgetter.h> - #include <vespa/searchcore/proton/server/replaypacketdispatcher.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/searchlib/transactionlog/translogserver.h> diff --git a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp index b700bf45820..716db497424 100644 --- a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp @@ -40,15 +40,24 @@ namespace proton namespace { -AttributesConfig::Attribute make_sv_cfg(AttributesConfig::Attribute::Datatype dataType) +AttributesConfig::Attribute make_sv_cfg(const vespalib::string &name, AttributesConfig::Attribute::Datatype dataType) { AttributesConfig::Attribute attr; - attr.name = "a"; + attr.name = name; attr.datatype = dataType; attr.collectiontype = AttributesConfig::Attribute::Collectiontype::SINGLE; return attr; } +AttributesConfig::Attribute make_sv_cfg(AttributesConfig::Attribute::Datatype dataType) +{ + return make_sv_cfg("a", dataType); +} + +AttributesConfig::Attribute make_int32_sv_cfg(const vespalib::string &name) { + return make_sv_cfg(name, AttributesConfig::Attribute::Datatype::INT32); +} + AttributesConfig::Attribute make_int32_sv_cfg() { return make_sv_cfg(AttributesConfig::Attribute::Datatype::INT32); } @@ -108,6 +117,14 @@ SummarymapConfig::Override make_geopos_override(const vespalib::string &name) return override; } +SummarymapConfig::Override make_attribute_combiner_override(const vespalib::string &name) +{ + SummarymapConfig::Override override; + override.field = name; + override.command = "attributecombiner"; + return override; +} + SummarymapConfig smCfg(std::vector<SummarymapConfig::Override> overrides) { SummarymapConfigBuilder result; @@ -310,6 +327,29 @@ TEST_F("require that fast access flag change is not delayed, true->false edge, s TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } +TEST_F("require that adding attribute aspect to struct field is not delayed if field type is changed", Fixture) +{ + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")})); + TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg("array.a")})); + TEST_DO(f.assertSummarymapConfig({make_attribute_combiner_override("array")})); +} + +TEST_F("require that adding attribute aspect to struct field is delayed if field type is unchanged", Fixture) +{ + f.addFields({"array.a"}); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")})); + TEST_DO(f.assertAttributeConfig({})); + TEST_DO(f.assertSummarymapConfig({})); +} + +TEST_F("require that removing attribute aspect from struct field is not delayed", Fixture) +{ + f.addFields({"array.a"}); + f.setup(attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")}), attrCfg({}), smCfg({})); + TEST_DO(f.assertAttributeConfig({})); + TEST_DO(f.assertSummarymapConfig({})); +} + } TEST_MAIN() diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 9bf4d43fff7..ce09a9a3742 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -1,7 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("attribute_manager_test"); - #include <vespa/config-attributes.h> #include <vespa/fastos/file.h> #include <vespa/searchcommon/attribute/attributecontent.h> @@ -42,6 +39,9 @@ LOG_SETUP("attribute_manager_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_manager_test"); + namespace vespa { namespace config { namespace search {}}} using std::string; @@ -84,7 +84,7 @@ class MyAttributeFunctor : public proton::IAttributeFunctor std::vector<vespalib::string> _names; public: - virtual void + void operator()(const search::AttributeVector &attributeVector) override { _names.push_back(attributeVector.getName()); } @@ -166,13 +166,12 @@ BaseFixture::BaseFixture() _hwInfo() { } -BaseFixture::~BaseFixture() {} +BaseFixture::~BaseFixture() = default; struct AttributeManagerFixture { proton::AttributeManager::SP _msp; proton::AttributeManager &_m; - AttributeWriter _aw; ImportedAttributesRepoBuilder _builder; AttributeManagerFixture(BaseFixture &bf); ~AttributeManagerFixture(); @@ -191,10 +190,9 @@ AttributeManagerFixture::AttributeManagerFixture(BaseFixture &bf) : _msp(std::make_shared<proton::AttributeManager>(test_dir, "test.subdb", TuneFileAttributes(), bf._fileHeaderContext, bf._attributeFieldWriter, bf._hwInfo)), _m(*_msp), - _aw(_msp), _builder() {} -AttributeManagerFixture::~AttributeManagerFixture() {} +AttributeManagerFixture::~AttributeManagerFixture() = default; struct Fixture : public BaseFixture, public AttributeManagerFixture { @@ -318,16 +316,21 @@ TEST_F("require that attributes are flushed and loaded", BaseFixture) fillAttribute(a1, 1, 2, 200); EXPECT_EQUAL(4u, a1->getNumDocs()); AttributeVector::SP a2 = amf.addAttribute("a2"); // loaded - EXPECT_EQUAL(5u, a2->getNumDocs()); - EXPECT_EQUAL(4u, a1->getNumDocs()); - amf._aw.onReplayDone(5u); - EXPECT_EQUAL(5u, a2->getNumDocs()); - EXPECT_EQUAL(5u, a1->getNumDocs()); - fillAttribute(a2, 1, 4, 200); - EXPECT_EQUAL(6u, a2->getNumDocs()); + { + AttributeWriter aw(amf._msp); + + EXPECT_EQUAL(5u, a2->getNumDocs()); + EXPECT_EQUAL(4u, a1->getNumDocs()); + aw.onReplayDone(5u); + EXPECT_EQUAL(5u, a2->getNumDocs()); + EXPECT_EQUAL(5u, a1->getNumDocs()); + fillAttribute(a2, 1, 4, 200); + EXPECT_EQUAL(6u, a2->getNumDocs()); + } AttributeVector::SP a3 = amf.addAttribute("a3"); // not-loaded + AttributeWriter aw(amf._msp); EXPECT_EQUAL(1u, a3->getNumDocs()); - amf._aw.onReplayDone(6); + aw.onReplayDone(6); EXPECT_EQUAL(6u, a3->getNumDocs()); fillAttribute(a3, 1, 7, 6, 200); EXPECT_EQUAL(7u, a3->getNumDocs()); @@ -352,10 +355,11 @@ TEST_F("require that attributes are flushed and loaded", BaseFixture) EXPECT_EQUAL(6u, a1->getNumDocs()); EXPECT_EQUAL(6u, a2->getNumDocs()); AttributeVector::SP a3 = amf.addAttribute("a3"); // loaded + AttributeWriter aw(amf._msp); EXPECT_EQUAL(6u, a1->getNumDocs()); EXPECT_EQUAL(6u, a2->getNumDocs()); EXPECT_EQUAL(7u, a3->getNumDocs()); - amf._aw.onReplayDone(7); + aw.onReplayDone(7); EXPECT_EQUAL(7u, a1->getNumDocs()); EXPECT_EQUAL(7u, a2->getNumDocs()); EXPECT_EQUAL(7u, a3->getNumDocs()); @@ -559,6 +563,7 @@ TEST_F("require that lid space can be compacted", Fixture) AttributeVector::SP a2 = f.addAttribute("a2"); AttributeVector::SP ex(new Int32Attribute("ex")); f._m.addExtraAttribute(ex); + AttributeWriter aw(f._msp); const int64_t attrValue = 33; fillAttribute(a1, 20, attrValue, 100); fillAttribute(a2, 20, attrValue, 100); @@ -571,7 +576,7 @@ TEST_F("require that lid space can be compacted", Fixture) EXPECT_EQUAL(21u, a2->getCommittedDocIdLimit()); EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit()); - f._aw.compactLidSpace(10, 101); + aw.compactLidSpace(10, 101); EXPECT_EQUAL(21u, a1->getNumDocs()); EXPECT_EQUAL(21u, a2->getNumDocs()); @@ -587,6 +592,7 @@ TEST_F("require that lid space compaction op can be ignored", Fixture) AttributeVector::SP a2 = f.addAttribute("a2"); AttributeVector::SP ex(new Int32Attribute("ex")); f._m.addExtraAttribute(ex); + AttributeWriter aw(f._msp); const int64_t attrValue = 33; fillAttribute(a1, 20, attrValue, 200); fillAttribute(a2, 20, attrValue, 100); @@ -599,7 +605,7 @@ TEST_F("require that lid space compaction op can be ignored", Fixture) EXPECT_EQUAL(21u, a2->getCommittedDocIdLimit()); EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit()); - f._aw.compactLidSpace(10, 101); + aw.compactLidSpace(10, 101); EXPECT_EQUAL(21u, a1->getNumDocs()); EXPECT_EQUAL(21u, a2->getNumDocs()); diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 5954932bbaf..346f94180b5 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -445,7 +445,7 @@ TEST_F("require that attribute writer handles update", Fixture) schema.addAttributeField(Schema::AttributeField("a2", schema::DataType::INT32, CollectionType::SINGLE)); DocBuilder idb(schema); const document::DocumentType &dt(idb.getDocumentType()); - DocumentUpdate upd(dt, DocumentId("doc::1")); + DocumentUpdate upd(*idb.getDocumentTypeRepo(), dt, DocumentId("doc::1")); upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 5))); upd.addUpdate(FieldUpdate(upd.getType().getField("a2")) @@ -489,7 +489,7 @@ TEST_F("require that attribute writer handles predicate update", Fixture) EXPECT_EQUAL(2u, a1->getNumDocs()); const document::DocumentType &dt(idb.getDocumentType()); - DocumentUpdate upd(dt, DocumentId("doc::1")); + DocumentUpdate upd(*idb.getDocumentTypeRepo(), dt, DocumentId("doc::1")); PredicateFieldValue new_value(builder.feature("foo").value("bar").build()); upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(AssignValueUpdate(new_value))); @@ -678,7 +678,7 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture) EXPECT_TRUE(tensor->equals(*tensor2)); const document::DocumentType &dt(builder.getDocumentType()); - DocumentUpdate upd(dt, DocumentId("doc::1")); + DocumentUpdate upd(*builder.getDocumentTypeRepo(), dt, DocumentId("doc::1")); auto new_tensor = createTensor({ {{{"x", "8"}, {"y", "9"}}, 11} }, {"x", "y"}); TensorFieldValue new_value; diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp index d73cd805af1..8a5f58bd7e0 100644 --- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp @@ -256,13 +256,12 @@ BaseFixture::BaseFixture(const HwInfo &hwInfo) _attributeFieldWriter(), _hwInfo(hwInfo) {} -BaseFixture::~BaseFixture() {} +BaseFixture::~BaseFixture() = default; struct AttributeManagerFixture { AttributeManager::SP _msp; AttributeManager &_m; - AttributeWriter _aw; AttributeManagerFixture(BaseFixture &bf); ~AttributeManagerFixture(); AttributeVector::SP addAttribute(const vespalib::string &name) { @@ -278,10 +277,9 @@ struct AttributeManagerFixture AttributeManagerFixture::AttributeManagerFixture(BaseFixture &bf) : _msp(std::make_shared<AttributeManager>(test_dir, "test.subdb", TuneFileAttributes(), bf._fileHeaderContext, bf._attributeFieldWriter, bf._hwInfo)), - _m(*_msp), - _aw(_msp) + _m(*_msp) {} -AttributeManagerFixture::~AttributeManagerFixture() {} +AttributeManagerFixture::~AttributeManagerFixture() = default; struct Fixture : public BaseFixture, public AttributeManagerFixture { @@ -530,6 +528,7 @@ Test::requireThatShrinkWorks() Fixture f; AttributeManager &am = f._m; AttributeVector::SP av = f.addAttribute("a10"); + AttributeWriter aw(f._msp); av->addDocs(1000 - av->getNumDocs()); av->commit(50, 50); @@ -546,13 +545,13 @@ Test::requireThatShrinkWorks() EXPECT_FALSE(av->canShrinkLidSpace()); EXPECT_EQUAL(1000u, av->getNumDocs()); EXPECT_EQUAL(100u, av->getCommittedDocIdLimit()); - f._aw.heartBeat(51); + aw.heartBeat(51); EXPECT_TRUE(av->wantShrinkLidSpace()); EXPECT_FALSE(av->canShrinkLidSpace()); EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(), ft->getApproxMemoryGain().getAfter()); g.reset(); - f._aw.heartBeat(52); + aw.heartBeat(52); EXPECT_TRUE(av->wantShrinkLidSpace()); EXPECT_TRUE(av->canShrinkLidSpace()); EXPECT_TRUE(ft->getApproxMemoryGain().getBefore() > diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp index 6d69d9b225b..c5ae0f97875 100644 --- a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp +++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp @@ -37,7 +37,7 @@ TEST_F("require that attribute write thread is blocked while guard is held", Fix { ReadGuard::UP guard = f.accessor.takeGuard(); Gate gate; - f.writer.execute("myattr", [&gate]() { gate.countDown(); }); + f.writer.execute(f.writer.getExecutorId(f.attribute->getNamePrefix()), [&gate]() { gate.countDown(); }); bool reachedZero = gate.await(100); EXPECT_FALSE(reachedZero); EXPECT_EQUAL(1u, gate.getCount()); diff --git a/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp b/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp index 2dbff7d40dd..167865b5c68 100644 --- a/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp +++ b/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp @@ -1,78 +1,131 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcore/proton/common/document_type_inspector.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/repo/configbuilder.h> -#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/base/field.h> +#include <vespa/document/datatype/datatypes.h> using namespace document; using namespace proton; -using document::config_builder::DocumenttypesConfigBuilderHelper; -using document::config_builder::Struct; -const int32_t doc_type_id = 787121340; -const vespalib::string type_name = "test"; -const vespalib::string header_name = type_name + ".header"; -const vespalib::string body_name = type_name + ".body"; - -std::unique_ptr<const DocumentTypeRepo> -makeOldDocTypeRepo() +template <class Type> +void +addFields(Type &type, bool fieldF3IsString, bool hasFieldF4, bool hasFieldF5) { - DocumenttypesConfigBuilderHelper builder; - builder.document(doc_type_id, type_name, - Struct(header_name), Struct(body_name). - addField("f1", DataType::T_STRING). - addField("f2", DataType::T_STRING). - addField("f3", DataType::T_STRING). - addField("f4", DataType::T_STRING)); - return std::unique_ptr<const DocumentTypeRepo>(new DocumentTypeRepo(builder.config())); + type.addField(Field("f1", 1, *DataType::STRING, true)); + type.addField(Field("f2", 2, *DataType::STRING, true)); + type.addField(Field("f3", 3, fieldF3IsString ? *DataType::STRING : *DataType::INT, true)); + if (hasFieldF4) { + type.addField(Field("f4", 4, *DataType::STRING, true)); + } + if (hasFieldF5) { + type.addField(Field("f5", 5, *DataType::STRING, true)); + } } -std::unique_ptr<const DocumentTypeRepo> -makeNewDocTypeRepo() +struct DocumentTypeFixture +{ + DocumentType _documentType; + StructDataType _structFieldType; + ArrayDataType _structArrayFieldType; + MapDataType _structMapFieldType; + MapDataType _mapFieldType; + + DocumentTypeFixture(bool fieldF3IsString, bool hasFieldF4, bool hasFieldF5, bool hasStruct, bool mapKeyIsByte); + ~DocumentTypeFixture(); +}; + +DocumentTypeFixture::DocumentTypeFixture(bool fieldF3IsString, bool hasFieldF4, bool hasFieldF5, bool hasStruct, bool mapKeyIsByte) + : _documentType("test"), + _structFieldType("struct"), + _structArrayFieldType(_structFieldType), + _structMapFieldType(mapKeyIsByte ? *DataType::BYTE : *DataType::STRING, _structFieldType), + _mapFieldType(mapKeyIsByte ? *DataType::BYTE : *DataType::STRING, *DataType::STRING) { - DocumenttypesConfigBuilderHelper builder; - builder.document(doc_type_id, type_name, - Struct(header_name), Struct(body_name). - addField("f1", DataType::T_STRING). - addField("f2", DataType::T_STRING). - addField("f3", DataType::T_INT). - addField("f5", DataType::T_STRING)); - return std::unique_ptr<const DocumentTypeRepo>(new DocumentTypeRepo(builder.config())); + addFields(_documentType, fieldF3IsString, hasFieldF4, hasFieldF5); + if (hasStruct) { + addFields(_structFieldType, fieldF3IsString, hasFieldF4, hasFieldF5); + _documentType.addField(Field("sarray", 11, _structArrayFieldType, true)); + _documentType.addField(Field("smap", 12, _structMapFieldType, true)); + _documentType.addField(Field("map", 13, _mapFieldType, true)); + } } +DocumentTypeFixture::~DocumentTypeFixture() = default; + struct Fixture { - std::unique_ptr<const DocumentTypeRepo> _oldRepo; - std::unique_ptr<const DocumentTypeRepo> _newRepo; + DocumentTypeFixture _oldDocType; + DocumentTypeFixture _newDocType; DocumentTypeInspector _inspector; - Fixture() - : _oldRepo(makeOldDocTypeRepo()), - _newRepo(makeNewDocTypeRepo()), - _inspector(*_oldRepo->getDocumentType("test"), *_newRepo->getDocumentType("test")) + Fixture(bool hasStruct = true, bool mapKeyIsByte = false) + : _oldDocType(true, true, false, hasStruct, mapKeyIsByte), + _newDocType(false, false, true, true, false), + _inspector(_oldDocType._documentType, _newDocType._documentType) { } }; TEST_F("require that unchanged fields are known", Fixture) { - EXPECT_TRUE(f._inspector.hasUnchangedField("f1")); - EXPECT_TRUE(f._inspector.hasUnchangedField("f2")); + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_TRUE(inspector.hasUnchangedField("f1")); + EXPECT_TRUE(inspector.hasUnchangedField("f2")); + EXPECT_TRUE(inspector.hasUnchangedField("sarray.f1")); + EXPECT_TRUE(inspector.hasUnchangedField("sarray.f2")); + EXPECT_TRUE(inspector.hasUnchangedField("smap.key")); + EXPECT_TRUE(inspector.hasUnchangedField("smap.value.f1")); + EXPECT_TRUE(inspector.hasUnchangedField("smap.value.f2")); + EXPECT_TRUE(inspector.hasUnchangedField("map.key")); + EXPECT_TRUE(inspector.hasUnchangedField("map.value")); } TEST_F("require that changed fields are detected", Fixture) { - EXPECT_FALSE(f._inspector.hasUnchangedField("f3")); + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_FALSE(inspector.hasUnchangedField("f3")); + EXPECT_FALSE(inspector.hasUnchangedField("sarray.f3")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f3")); } TEST_F("require that partially missing fields are detected", Fixture) { - EXPECT_FALSE(f._inspector.hasUnchangedField("f4")); - EXPECT_FALSE(f._inspector.hasUnchangedField("f5")); + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_FALSE(inspector.hasUnchangedField("f4")); + EXPECT_FALSE(inspector.hasUnchangedField("f5")); + EXPECT_FALSE(inspector.hasUnchangedField("sarray.f4")); + EXPECT_FALSE(inspector.hasUnchangedField("sarray.f5")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f4")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f5")); } TEST_F("require that non-existing fields are NOT known", Fixture) { - EXPECT_FALSE(f._inspector.hasUnchangedField("not")); + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_FALSE(inspector.hasUnchangedField("not")); + EXPECT_FALSE(inspector.hasUnchangedField("sarray.not")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.not")); +} + +TEST_F("require that map key type change is detected", Fixture(true, true)) +{ + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_FALSE(inspector.hasUnchangedField("smap.key")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f1")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f2")); + EXPECT_FALSE(inspector.hasUnchangedField("map.key")); + EXPECT_FALSE(inspector.hasUnchangedField("map.value")); +} + +TEST_F("require that struct addition is detected", Fixture(false, false)) +{ + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_FALSE(inspector.hasUnchangedField("sarray.f1")); + EXPECT_FALSE(inspector.hasUnchangedField("sarray.f2")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.key")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f1")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f2")); + EXPECT_FALSE(inspector.hasUnchangedField("map.key")); + EXPECT_FALSE(inspector.hasUnchangedField("map.value")); } TEST_MAIN() diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 79d38e2dc56..1c901097812 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -829,20 +829,16 @@ Test::requireThatAttributesAreUsed() "bi:[]}", *rep, 1, false)); TEST_DO(assertTensor(Tensor::UP(), "bj", *rep, 1, rclass)); - proton::IAttributeManager::SP attributeManager = - dc._ddb->getReadySubDB()->getAttributeManager(); - search::ISequencedTaskExecutor &attributeFieldWriter = - attributeManager->getAttributeFieldWriter(); - search::AttributeVector *bjAttr = - attributeManager->getWritableAttribute("bj"); - search::tensor::TensorAttribute *bjTensorAttr = - dynamic_cast<search::tensor::TensorAttribute *>(bjAttr); - - attributeFieldWriter. - execute("bj", - [&]() { bjTensorAttr->setTensor(3, - *createTensor({ {{{"x", "a"},{"y", "b"}}, 4} }, { "x"})); - bjTensorAttr->commit(); }); + proton::IAttributeManager::SP attributeManager = dc._ddb->getReadySubDB()->getAttributeManager(); + search::ISequencedTaskExecutor &attributeFieldWriter = attributeManager->getAttributeFieldWriter(); + search::AttributeVector *bjAttr = attributeManager->getWritableAttribute("bj"); + auto bjTensorAttr = dynamic_cast<search::tensor::TensorAttribute *>(bjAttr); + + attributeFieldWriter.execute(attributeFieldWriter.getExecutorId(bjAttr->getNamePrefix()), + [&]() { + bjTensorAttr->setTensor(3, *createTensor({ {{{"x", "a"},{"y", "b"}}, 4} }, { "x"})); + bjTensorAttr->commit(); + }); attributeFieldWriter.sync(); DocsumReply::UP rep2 = dc._ddb->getDocsums(req); @@ -961,8 +957,7 @@ Test::requireThatUrisAreUsed() Document::UP exp = bc._bld.startDocument("doc::0"). startIndexField("urisingle"). startSubField("all"). - addUrlTokenizedString( - "http://www.example.com:81/fluke?ab=2#4"). + addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). @@ -986,8 +981,7 @@ Test::requireThatUrisAreUsed() startIndexField("uriarray"). startElement(1). startSubField("all"). - addUrlTokenizedString( - "http://www.example.com:82/fluke?ab=2#8"). + addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). @@ -1010,8 +1004,7 @@ Test::requireThatUrisAreUsed() endElement(). startElement(1). startSubField("all"). - addUrlTokenizedString( - "http://www.flickr.com:82/fluke?ab=2#9"). + addUrlTokenizedString("http://www.flickr.com:82/fluke?ab=2#9"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). @@ -1036,8 +1029,7 @@ Test::requireThatUrisAreUsed() startIndexField("uriwset"). startElement(4). startSubField("all"). - addUrlTokenizedString( - "http://www.example.com:83/fluke?ab=2#12"). + addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). @@ -1060,8 +1052,7 @@ Test::requireThatUrisAreUsed() endElement(). startElement(7). startSubField("all"). - addUrlTokenizedString( - "http://www.flickr.com:85/fluke?ab=2#13"). + addUrlTokenizedString("http://www.flickr.com:85/fluke?ab=2#13"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). diff --git a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp index 7f8ce99beb9..73d071be96f 100644 --- a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp +++ b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp @@ -33,7 +33,7 @@ #include <vespa/document/fieldvalue/referencefieldvalue.h> #include <vespa/document/predicate/predicate.h> #include <vespa/document/repo/configbuilder.h> -#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchsummary/docsummary/summaryfieldconverter.h> #include <vespa/searchsummary/docsummary/linguisticsannotation.h> #include <vespa/searchsummary/docsummary/searchdatatype.h> diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp index f06bb124eb8..958be6a4686 100644 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp @@ -1,15 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("combiningfeedview_test"); #include <vespa/document/test/make_bucket_space.h> -#include <vespa/searchcore/proton/feedoperation/moveoperation.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/server/combiningfeedview.h> #include <vespa/searchcore/proton/test/test.h> #include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/document/update/documentupdate.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("combiningfeedview_test"); + using document::DocumentTypeRepo; using document::DocumentUpdate; using document::test::makeBucketSpace; @@ -22,8 +23,8 @@ typedef std::vector<IFeedView::SP> FeedViewVector; struct MyStreamHandler : public NewConfigOperation::IStreamHandler { - virtual void serializeConfig(SerialNum, vespalib::nbostream &) override {} - virtual void deserializeConfig(SerialNum, vespalib::nbostream &) override {} + void serializeConfig(SerialNum, vespalib::nbostream &) override {} + void deserializeConfig(SerialNum, vespalib::nbostream &) override {} }; diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index bab6aff821f..760f7457d33 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -14,6 +14,7 @@ #include <vespa/searchcore/proton/reference/i_document_db_reference_resolver.h> #include <vespa/searchcore/proton/reprocessing/i_reprocessing_task.h> #include <vespa/searchcore/proton/reprocessing/reprocessingrunner.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/document_subdb_explorer.h> #include <vespa/searchcore/proton/server/emptysearchview.h> diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index e774728b41e..a1bce7174bf 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -316,7 +316,7 @@ struct UpdateContext { DocumentUpdate::SP update; BucketId bucketId; UpdateContext(const vespalib::string &docId, DocBuilder &builder) : - update(new DocumentUpdate(builder.getDocumentType(), DocumentId(docId))), + update(new DocumentUpdate(*builder.getDocumentTypeRepo(), builder.getDocumentType(), DocumentId(docId))), bucketId(BucketFactory::getBucketId(update->getId())) { } diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index a9fa79f513c..d21e5abe30b 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -11,6 +11,7 @@ #include <vespa/searchcore/proton/server/isummaryadapter.h> #include <vespa/searchcore/proton/server/matchview.h> #include <vespa/searchcore/proton/server/searchable_feed_view.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/test/document_meta_store_context_observer.h> #include <vespa/searchcore/proton/test/dummy_document_store.h> #include <vespa/searchcore/proton/test/dummy_summary_manager.h> @@ -478,11 +479,11 @@ struct DocumentContext DocumentContext::DocumentContext(const vespalib::string &docId, uint64_t timestamp, DocBuilder &builder) : doc(builder.startDocument(docId).startSummaryField("s1").addStr(docId).endField().endDocument().release()), - upd(new DocumentUpdate(builder.getDocumentType(), doc->getId())), + upd(new DocumentUpdate(*builder.getDocumentTypeRepo(), builder.getDocumentType(), doc->getId())), bid(BucketFactory::getNumBucketBits(), doc->getId().getGlobalId().convertToBucketId().getRawId()), ts(timestamp) {} -DocumentContext::~DocumentContext() {} +DocumentContext::~DocumentContext() = default; struct FeedTokenContext { diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index e8247a52199..d07c29ead69 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -9,6 +9,8 @@ #include <vespa/searchcore/proton/server/putdonecontext.h> #include <vespa/searchcore/proton/server/removedonecontext.h> #include <vespa/searchcore/proton/server/storeonlyfeedview.h> +#include <vespa/searchcore/proton/feedoperation/moveoperation.h> +#include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h> #include <vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h> #include <vespa/searchcore/proton/test/mock_summary_adapter.h> #include <vespa/searchcore/proton/test/thread_utils.h> diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp index 9a9d896f2b9..e87e9209a17 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp @@ -122,7 +122,7 @@ public: } auto makeUpdate() { - auto upd(std::make_shared<DocumentUpdate>(_docType, docId)); + auto upd(std::make_shared<DocumentUpdate>(*_repo, _docType, docId)); upd->addUpdate(FieldUpdate(upd->getType().getField("string")). addUpdate(AssignValueUpdate(StringFieldValue("newval")))); return upd; @@ -136,6 +136,7 @@ public: TEST("require that toString() on derived classes are meaningful") { + DocumentTypeRepo repo; BucketId bucket_id1(42); BucketId bucket_id2(43); BucketId bucket_id3(44); @@ -146,7 +147,7 @@ TEST("require that toString() on derived classes are meaningful") MyStreamHandler stream_handler; DocumentIdT doc_id_limit = 15; DocumentId doc_id("doc:foo:bar"); - DocumentUpdate::SP update(new DocumentUpdate(*DataType::DOCUMENT, doc_id)); + DocumentUpdate::SP update(new DocumentUpdate(repo, *DataType::DOCUMENT, doc_id)); EXPECT_EQUAL("DeleteBucket(BucketId(0x0000000000000000), serialNum=0)", DeleteBucketOperation().toString()); diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 73580416e65..7f065d0cc15 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -67,7 +67,9 @@ createDoc(const DocumentType &docType, const DocumentId &docId) document::DocumentUpdate::SP createUpd(const DocumentType& docType, const DocumentId &docId) { - return document::DocumentUpdate::SP(new document::DocumentUpdate(docType, docId)); + static std::vector<std::unique_ptr<document::DocumentTypeRepo>> repoList; + repoList.emplace_back(std::make_unique<document::DocumentTypeRepo>(docType)); + return std::make_shared<document::DocumentUpdate>(*repoList.back(), docType, docId); } storage::spi::ClusterState diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp index 93c192b57e8..7be774f7291 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp @@ -306,6 +306,18 @@ TEST("require that added attribute aspect with flushed attribute after interrupt EXPECT_TRUE(f.assertAttributes({})); } +TEST_F("require that removed attribute aspect from struct field does not require document field populate", Fixture) +{ + f.addOldConfig({"array.a"}, {"array.a"}).addNewConfig({"array.a"}, {}).init(); + EXPECT_TRUE(f.assertFields({})); +} + +TEST_F("require that added attribute aspect to struct field requires attribute populate", Fixture) +{ + f.addOldConfig({"array.a"}, {}).addNewConfig({"array.a"}, {"array.a"}).init(); + EXPECT_TRUE(f.assertAttributes({"array.a"})); +} + TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index dfa461e2b57..f206ffc9b17 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for feedstates. -#include <vespa/log/log.h> -LOG_SETUP("feedstates_test"); #include <vespa/document/base/documentid.h> #include <vespa/document/base/testdocrepo.h> @@ -11,6 +9,7 @@ LOG_SETUP("feedstates_test"); #include <vespa/searchcore/proton/server/feedstates.h> #include <vespa/searchcore/proton/server/ireplayconfig.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> +#include <vespa/searchcore/proton/feedoperation/removeoperation.h> #include <vespa/searchcore/proton/test/dummy_feed_view.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/objects/nbostream.h> @@ -18,6 +17,9 @@ LOG_SETUP("feedstates_test"); #include <vespa/vespalib/util/buffer.h> #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> +#include <vespa/log/log.h> +LOG_SETUP("feedstates_test"); + using document::BucketId; using document::DocumentId; using document::DocumentTypeRepo; @@ -83,7 +85,7 @@ Fixture::Fixture() state("doctypename", feed_view_ptr, _bucketDBHandler, replay_config, config_store) { } -Fixture::~Fixture() {} +Fixture::~Fixture() = default; struct RemoveOperationContext @@ -107,7 +109,7 @@ RemoveOperationContext::RemoveOperationContext(search::SerialNum serial) packet.reset(new Packet()); packet->add(Packet::Entry(serial, FeedOperation::REMOVE, buf)); } -RemoveOperationContext::~RemoveOperationContext() {} +RemoveOperationContext::~RemoveOperationContext() = default; TEST_F("require that active FeedView can change during replay", Fixture) { RemoveOperationContext opCtx(10); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index e66466aa1cb..7578066d93a 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -467,7 +467,7 @@ hwinfo.cpu.cores int default = 0 restart ## max(ceil(hwinfo.cpu.cores * feeding.concurrency), summary.log.numthreads) ## The number of threads in each of pools 2-4 is calculated as: ## max(ceil((hwinfo.cpu.cores * feeding.concurrency)/3), indexing.threads) -feeding.concurrency double default = 0.5 restart +feeding.concurrency double default = 0.2 restart ## Adjustment to resource limit when determining if maintenance jobs can run. ## diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp index 9807faa5021..cf803ec0368 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp @@ -29,11 +29,15 @@ bool fastPartialUpdateAttribute(const search::attribute::Config &cfg) { (basicType != BasicType::Type::REFERENCE)); } +bool isStructFieldAttribute(const vespalib::string &name) { + return name.find('.') != vespalib::string::npos; +} + bool willTriggerReprocessOnAttributeAspectRemoval(const search::attribute::Config &cfg, const IIndexschemaInspector &indexschemaInspector, const vespalib::string &name) { - return fastPartialUpdateAttribute(cfg) && !indexschemaInspector.isStringIndex(name); + return fastPartialUpdateAttribute(cfg) && !indexschemaInspector.isStringIndex(name) && !isStructFieldAttribute(name); } @@ -73,6 +77,7 @@ handleNewAttributes(const AttributesConfig &oldAttributesConfig, SummarymapConfigBuilder &summarymapConfig) { vespalib::hash_set<vespalib::string> delayed; + vespalib::hash_set<vespalib::string> delayedStruct; AttributesConfigHash oldAttrs(oldAttributesConfig.attribute); for (const auto &newAttr : newAttributesConfig.attribute) { search::attribute::Config newCfg = ConfigConverter::convert(newAttr); @@ -102,6 +107,10 @@ handleNewAttributes(const AttributesConfig &oldAttributesConfig, } else { // Delay addition of attribute aspect delayed.insert(newAttr.name); + auto pos = newAttr.name.find('.'); + if (pos != vespalib::string::npos) { + delayedStruct.insert(newAttr.name.substr(0, pos)); + } } } } @@ -111,6 +120,11 @@ handleNewAttributes(const AttributesConfig &oldAttributesConfig, if (itr == delayed.end()) { summarymapConfig.override.emplace_back(override); } + } else if (override.command == "attributecombiner") { + auto itr = delayedStruct.find(override.field); + if (itr == delayedStruct.end()) { + summarymapConfig.override.emplace_back(override); + } } else { summarymapConfig.override.emplace_back(override); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 0a0ecbda21c..506f4b5164c 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -7,16 +7,16 @@ LOG_SETUP(".proton.attribute.attribute_writer"); #include "ifieldupdatecallback.h" #include "attributemanager.h" #include "document_field_extractor.h" -#include <vespa/document/base/exceptions.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/fieldvalue/document.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/attrupdate.h> #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/searchlib/common/idestructorcallback.h> - +#include <vespa/document/base/exceptions.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/vespalib/stllike/hash_map.hpp> using namespace document; using namespace search; @@ -47,6 +47,8 @@ AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType) docType.buildFieldPath(fp, name); } catch (document::FieldNotFoundException & e) { fp = FieldPath(); + } catch (vespalib::IllegalArgumentException &e) { + fp = FieldPath(); } _fieldPath = std::move(fp); } @@ -230,7 +232,7 @@ public: FieldContext::FieldContext(ISequencedTaskExecutor &writer, AttributeVector *attr) : _name(attr->getName()), - _executorId(writer.getExecutorId(_name)), + _executorId(writer.getExecutorId(attr->getNamePrefix())), _attr(attr) { } @@ -412,7 +414,7 @@ AttributeWriter::setupWriteContexts() { std::vector<FieldContext> fieldContexts; assert(_writeContexts.empty()); - for (auto attr : _writableAttributes) { + for (auto attr : getWritableAttributes()) { fieldContexts.emplace_back(_attributeFieldWriter, attr); } std::sort(fieldContexts.begin(), fieldContexts.end()); @@ -469,14 +471,23 @@ AttributeWriter::internalRemove(SerialNum serialNum, DocumentIdT lid, bool immed AttributeWriter::AttributeWriter(const proton::IAttributeManager::SP &mgr) : _mgr(mgr), _attributeFieldWriter(mgr->getAttributeFieldWriter()), - _writableAttributes(mgr->getWritableAttributes()), _writeContexts(), _dataType(nullptr), - _hasStructFieldAttribute(false) + _hasStructFieldAttribute(false), + _attrMap() { setupWriteContexts(); + setupAttriuteMapping(); } +void AttributeWriter::setupAttriuteMapping() { + for (auto attr : getWritableAttributes()) { + vespalib::stringref name = attr->getName(); + _attrMap[name] = AttrWithId(attr, _attributeFieldWriter.getExecutorId(attr->getNamePrefix())); + } +} + + AttributeWriter::~AttributeWriter() { _attributeFieldWriter.sync(); @@ -545,7 +556,8 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document for (const auto &fupd : upd.getUpdates()) { LOG(debug, "Retrieving guard for attribute vector '%s'.", fupd.getField().getName().c_str()); - AttributeVector *attrp = _mgr->getWritableAttribute(fupd.getField().getName()); + auto found = _attrMap.find(fupd.getField().getName()); + AttributeVector * attrp = (found != _attrMap.end()) ? found->second.first : nullptr; onUpdate.onUpdateField(fupd.getField().getName(), attrp); if (attrp == nullptr) { LOG(spam, "Failed to find attribute vector %s", fupd.getField().getName().c_str()); @@ -555,7 +567,7 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document // document and attribute. if (attrp->getStatus().getLastSyncToken() >= serialNum) continue; - args[_attributeFieldWriter.getExecutorId(attrp->getName()).getId()]->_updates.emplace_back(attrp, &fupd); + args[found->second.second.getId()]->_updates.emplace_back(attrp, &fupd); LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", lid, attrp->getName().c_str()); } // NOTE: The lifetime of the field update will be ensured by keeping the document update alive @@ -572,11 +584,10 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document void AttributeWriter::heartBeat(SerialNum serialNum) { - for (auto attrp : _writableAttributes) { - auto &attr = *attrp; - _attributeFieldWriter.execute(attr.getName(), - [serialNum, &attr]() - { applyHeartBeat(serialNum, attr); }); + for (auto entry : _attrMap) { + _attributeFieldWriter.execute(entry.second.second, + [serialNum, attr=entry.second.first]() + { applyHeartBeat(serialNum, *attr); }); } } @@ -601,11 +612,10 @@ AttributeWriter::forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) void AttributeWriter::onReplayDone(uint32_t docIdLimit) { - for (auto attrp : _writableAttributes) { - auto &attr = *attrp; - _attributeFieldWriter.execute(attr.getName(), - [docIdLimit, &attr]() - { applyReplayDone(docIdLimit, attr); }); + for (auto entry : _attrMap) { + _attributeFieldWriter.execute(entry.second.second, + [docIdLimit, attr = entry.second.first]() + { applyReplayDone(docIdLimit, *attr); }); } _attributeFieldWriter.sync(); } @@ -614,12 +624,11 @@ AttributeWriter::onReplayDone(uint32_t docIdLimit) void AttributeWriter::compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) { - for (auto attrp : _writableAttributes) { - auto &attr = *attrp; + for (auto entry : _attrMap) { _attributeFieldWriter. - execute(attr.getName(), - [wantedLidLimit, serialNum, &attr]() - { applyCompactLidSpace(wantedLidLimit, serialNum, attr); }); + execute(entry.second.second, + [wantedLidLimit, serialNum, attr=entry.second.first]() + { applyCompactLidSpace(wantedLidLimit, serialNum, *attr); }); } _attributeFieldWriter.sync(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index f89089ed335..4ea7f3fda6c 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -6,6 +6,7 @@ #include <vespa/searchcore/proton/common/commit_time_tracker.h> #include <vespa/document/base/fieldpath.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> +#include <vespa/vespalib/stllike/hash_map.h> namespace document { class DocumentType; } @@ -25,7 +26,6 @@ private: typedef document::FieldValue FieldValue; const IAttributeManager::SP _mgr; search::ISequencedTaskExecutor &_attributeFieldWriter; - const std::vector<search::AttributeVector *> &_writableAttributes; using ExecutorId = search::ISequencedTaskExecutor::ExecutorId; public: class WriteField @@ -58,11 +58,15 @@ public: bool hasStructFieldAttribute() const { return _hasStructFieldAttribute; } }; private: + using AttrWithId = std::pair<search::AttributeVector *, ExecutorId>; + using AttrMap = vespalib::hash_map<vespalib::string, AttrWithId>; std::vector<WriteContext> _writeContexts; const DataType *_dataType; bool _hasStructFieldAttribute; + AttrMap _attrMap; void setupWriteContexts(); + void setupAttriuteMapping(); void buildFieldPaths(const DocumentType &docType, const DataType *dataType); void internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, bool allAttributes, OnWriteDoneType onWriteDone); @@ -73,13 +77,13 @@ public: AttributeWriter(const proton::IAttributeManager::SP &mgr); ~AttributeWriter(); + /* Only for in tests that add attributes after AttributeWriter construction. */ + /** * Implements IAttributeWriter. */ - std::vector<search::AttributeVector *> - getWritableAttributes() const override; - search::AttributeVector * - getWritableAttribute(const vespalib::string &name) const override; + std::vector<search::AttributeVector *> getWritableAttributes() const override; + search::AttributeVector *getWritableAttribute(const vespalib::string &name) const override; void put(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) override; void remove(SerialNum serialNum, DocumentIdT lid, diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index ef818f7b407..5faf89aa149 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -78,8 +78,8 @@ std::shared_ptr<ShrinkLidSpaceFlushTarget> allocShrinker(const AttributeVector:: using Type = IFlushTarget::Type; using Component = IFlushTarget::Component; + auto shrinkwrap = std::make_shared<ThreadedCompactableLidSpace>(attr, attributeFieldWriter, attributeFieldWriter.getExecutorId(attr->getNamePrefix())); const vespalib::string &name = attr->getName(); - auto shrinkwrap = std::make_shared<ThreadedCompactableLidSpace>(attr, attributeFieldWriter, attributeFieldWriter.getExecutorId(name)); auto dir = diskLayout.createAttributeDir(name); search::SerialNum shrinkSerialNum = estimateShrinkSerialNum(*attr); return std::make_shared<ShrinkLidSpaceFlushTarget>("attribute.shrink." + name, Type::GC, Component::ATTRIBUTE, shrinkSerialNum, dir->getLastFlushTime(), shrinkwrap); @@ -569,16 +569,15 @@ AttributeManager::getWritableAttributes() const void -AttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> - func) const +AttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const { for (const auto &attr : _attributes) { if (attr.second.isExtra()) { continue; } AttributeVector::SP attrsp = attr.second.getAttribute(); - _attributeFieldWriter. - execute(attr.first, [attrsp, func]() { (*func)(*attrsp); }); + _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorId(attrsp->getNamePrefix()), + [attrsp, func]() { (*func)(*attrsp); }); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp index d9a0ff3d8dd..d1d5b1c9af7 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp @@ -37,8 +37,7 @@ ExclusiveAttributeReadAccessor(const AttributeVector::SP &attribute, namespace { void -attributeWriteBlockingTask(GateSP entranceGate, - GateSP exitGate) +attributeWriteBlockingTask(GateSP entranceGate, GateSP exitGate) { entranceGate->countDown(); exitGate->await(); @@ -51,9 +50,8 @@ ExclusiveAttributeReadAccessor::takeGuard() { GateSP entranceGate = std::make_shared<Gate>(); GateSP exitGate = std::make_shared<Gate>(); - _attributeFieldWriter.execute(_attribute->getName(), - [entranceGate, exitGate]() - { attributeWriteBlockingTask(entranceGate, exitGate); }); + _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorId(_attribute->getNamePrefix()), + [entranceGate, exitGate]() { attributeWriteBlockingTask(entranceGate, exitGate); }); entranceGate->await(); return std::make_unique<Guard>(*_attribute, exitGate); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index d3a74bb9a98..8474efb15c9 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -197,9 +197,8 @@ FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> search::AttributeVector::SP attrsp = guard.getSP(); // Name must be extracted in document db master thread or attribute // writer thread - vespalib::string attributeName = attrsp->getName(); - attributeFieldWriter. - execute(attributeName, [attrsp, func]() { (*func)(*attrsp); }); + attributeFieldWriter.execute(attributeFieldWriter.getExecutorId(attrsp->getNamePrefix()), + [attrsp, func]() { (*func)(*attrsp); }); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index a658b11263a..7716fc5ee61 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -243,10 +243,8 @@ FlushableAttribute::initFlush(SerialNum currentSerial) // Called by document db executor std::promise<IFlushTarget::Task::UP> promise; std::future<IFlushTarget::Task::UP> future = promise.get_future(); - _attributeFieldWriter.execute(_attr->getName(), - [&]() { promise.set_value( - internalInitFlush(currentSerial)); - }); + _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorId(_attr->getNamePrefix()), + [&]() { promise.set_value(internalInitFlush(currentSerial)); }); return future.get(); } @@ -257,5 +255,4 @@ FlushableAttribute::getApproxBytesToWriteToDisk() const return _attr->getEstimatedSaveByteSize(); } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp b/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp index 6cff162ae08..e19fa5351c2 100644 --- a/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp @@ -1,6 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_type_inspector.h" +#include <vespa/document/base/exceptions.h> +#include <vespa/document/base/fieldpath.h> + +using document::FieldPath; +using document::FieldPathEntry; namespace proton { @@ -14,12 +19,28 @@ DocumentTypeInspector::DocumentTypeInspector(const document::DocumentType &oldDo bool DocumentTypeInspector::hasUnchangedField(const vespalib::string &name) const { - if (!_oldDocType.hasField(name) || !_newDocType.hasField(name)) { + FieldPath oldPath; + FieldPath newPath; + try { + _oldDocType.buildFieldPath(oldPath, name); + _newDocType.buildFieldPath(newPath, name); + } catch (document::FieldNotFoundException &e) { + return false; + } catch (vespalib::IllegalArgumentException &e) { return false; } - const document::Field &oldField = _oldDocType.getField(name); - const document::Field &newField = _newDocType.getField(name); - return oldField == newField; + if (oldPath.size() != newPath.size()) { + return false; + } + for (uint32_t i = 0; i < oldPath.size(); ++i) { + const auto &oldEntry = oldPath[i]; + const auto &newEntry = newPath[i]; + if (oldEntry.getType() != newEntry.getType() || + oldEntry.getDataType() != newEntry.getDataType()) { + return false; + } + } + return true; } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp index cd906891b92..c2a3439388a 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "compact_lid_space_operation.h" +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/stringfmt.h> namespace proton { @@ -27,8 +28,7 @@ CompactLidSpaceOperation::serialize(vespalib::nbostream& os) const } void -CompactLidSpaceOperation::deserialize(vespalib::nbostream& is, - const document::DocumentTypeRepo&) +CompactLidSpaceOperation::deserialize(vespalib::nbostream& is, const document::DocumentTypeRepo&) { is >> _subDbId; is >> _lidLimit; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h index 8a00c739126..77b95547bd0 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h @@ -2,10 +2,10 @@ #pragma once #include <vespa/searchlib/common/serialnum.h> -#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/stllike/string.h> namespace document { class DocumentTypeRepo; } - +namespace vespalib { class nbostream; } namespace proton { class FeedOperation diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp index 71a701bdb40..67e4d1b4287 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp @@ -2,6 +2,7 @@ #include "lidvectorcontext.h" #include <vespa/searchlib/common/bitvector.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".proton.feedoperation.lidvectorcontext"); diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.h b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.h index b432bbcf20a..b307e50da0a 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.h @@ -2,9 +2,10 @@ #pragma once #include <vespa/searchlib/query/base.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vector> +namespace vespalib { class nbostream; } + namespace proton { class LidVectorContext @@ -29,5 +30,4 @@ public: size_t getNumLids() const { return _result.size(); } }; -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.h index b7b041135d4..144aa534bf7 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.h @@ -2,30 +2,29 @@ #pragma once #include "feedoperation.h" -#include <vespa/vespalib/objects/nbostream.h> namespace proton { + namespace feedoperation { + struct IStreamHandler { + virtual ~IStreamHandler() {} + virtual void serializeConfig(search::SerialNum serialNum, vespalib::nbostream &os) = 0; + virtual void deserializeConfig(search::SerialNum serialNum, vespalib::nbostream &is) = 0; + }; + } + class NewConfigOperation : public FeedOperation { public: - struct IStreamHandler { - virtual ~IStreamHandler() {} - virtual void serializeConfig(SerialNum serialNum, - vespalib::nbostream &os) = 0; - virtual void deserializeConfig(SerialNum serialNum, - vespalib::nbostream &is) = 0; - }; + using IStreamHandler = feedoperation::IStreamHandler; private: IStreamHandler &_streamHandler; public: - NewConfigOperation(SerialNum serialNum, - IStreamHandler &streamHandler); - virtual ~NewConfigOperation() {} - virtual void serialize(vespalib::nbostream &os) const override; - virtual void deserialize(vespalib::nbostream &is, - const document::DocumentTypeRepo &repo) override; - virtual vespalib::string toString() const override; + NewConfigOperation(SerialNum serialNum, IStreamHandler &streamHandler); + ~NewConfigOperation() override {} + void serialize(vespalib::nbostream &os) const override; + void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; + vespalib::string toString() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp index eea693e4680..69f754cd594 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp @@ -2,6 +2,7 @@ #include "pruneremoveddocumentsoperation.h" #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/objects/nbostream.h> #include <cassert> #include <vespa/log/log.h> @@ -22,8 +23,7 @@ PruneRemovedDocumentsOperation::PruneRemovedDocumentsOperation() PruneRemovedDocumentsOperation:: -PruneRemovedDocumentsOperation(DocumentIdT docIdLimit, - uint32_t subDbId) +PruneRemovedDocumentsOperation(DocumentIdT docIdLimit, uint32_t subDbId) : RemoveDocumentsOperation(FeedOperation::PRUNE_REMOVED_DOCUMENTS), _subDbId(subDbId) { @@ -44,8 +44,7 @@ PruneRemovedDocumentsOperation::serialize(vespalib::nbostream &os) const void -PruneRemovedDocumentsOperation::deserialize(vespalib::nbostream &is, - const DocumentTypeRepo &) +PruneRemovedDocumentsOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &) { is >> _subDbId; deserializeLidsToRemove(is); diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp index 6aa479611d1..ef482a19ca3 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "removedocumentsoperation.h" +#include <vespa/vespalib/objects/nbostream.h> namespace proton { @@ -16,11 +17,9 @@ RemoveDocumentsOperation::serializeLidsToRemove(vespalib::nbostream &os) const { uint32_t mapSize = _lidsToRemoveMap.size(); os << mapSize; - for (LidsToRemoveMap::const_iterator - it = _lidsToRemoveMap.begin(), ite = _lidsToRemoveMap.end(); - it != ite; ++it) { - os << it->first; - it->second->serialize(os); + for (const auto & entry : _lidsToRemoveMap) { + os << entry.first; + entry.second->serialize(os); } } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp index f3ef73f4d90..16ddedc4745 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "spoolerreplayoperation.h" +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/stringfmt.h> using vespalib::make_string; @@ -14,10 +15,7 @@ SpoolerReplayOperation::SpoolerReplayOperation(Type type) { } - -SpoolerReplayOperation::SpoolerReplayOperation(Type type, - SerialNum serialNum, - SerialNum spoolerSerialNum) +SpoolerReplayOperation::SpoolerReplayOperation(Type type, SerialNum serialNum, SerialNum spoolerSerialNum) : FeedOperation(type), _spoolerSerialNum(spoolerSerialNum) { @@ -39,11 +37,8 @@ SpoolerReplayOperation::deserialize(vespalib::nbostream &is) } vespalib::string SpoolerReplayOperation::toString() const { - return make_string( - "SpoolerReplay%s(spoolerSerialNum=%" PRIu64 - ", serialNum=%" PRIu64 ")", - getType() == SPOOLER_REPLAY_START ? "Start" : "Complete", - _spoolerSerialNum, getSerialNum()); + return make_string("SpoolerReplay%s(spoolerSerialNum=%" PRIu64", serialNum=%" PRIu64 ")", + getType() == SPOOLER_REPLAY_START ? "Start" : "Complete", _spoolerSerialNum, getSerialNum()); } @@ -53,8 +48,7 @@ SpoolerReplayStartOperation::SpoolerReplayStartOperation() } -SpoolerReplayStartOperation::SpoolerReplayStartOperation(SerialNum serialNum, - SerialNum spoolerSerialNum) +SpoolerReplayStartOperation::SpoolerReplayStartOperation(SerialNum serialNum, SerialNum spoolerSerialNum) : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_START, serialNum, spoolerSerialNum) @@ -70,12 +64,8 @@ SpoolerReplayCompleteOperation::SpoolerReplayCompleteOperation() SpoolerReplayCompleteOperation::SpoolerReplayCompleteOperation(SerialNum serialNum, SerialNum spoolerSerialNum) - : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_COMPLETE, - serialNum, - spoolerSerialNum) + : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_COMPLETE, serialNum, spoolerSerialNum) { } - - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h index 028d01dec20..028ad1c6bfa 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h @@ -11,15 +11,12 @@ private: SerialNum _spoolerSerialNum; protected: SpoolerReplayOperation(Type type); - SpoolerReplayOperation(Type type, - SerialNum serialNum, - SerialNum spoolerSerialNum); + SpoolerReplayOperation(Type type, SerialNum serialNum, SerialNum spoolerSerialNum); public: - virtual ~SpoolerReplayOperation() {} + ~SpoolerReplayOperation() override {} SerialNum getSpoolerSerialNum() const { return _spoolerSerialNum; } - virtual void serialize(vespalib::nbostream &os) const override; - virtual void deserialize(vespalib::nbostream &is, - const document::DocumentTypeRepo &) override { + void serialize(vespalib::nbostream &os) const override; + void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &) override { deserialize(is); } void deserialize(vespalib::nbostream &is); @@ -38,8 +35,7 @@ public: * @param serialNum the current serial number of the transaction log. * @param spoolerSerialNum the serial number of the first entry of the spooler log replay. */ - SpoolerReplayStartOperation(SerialNum serialNum, - SerialNum spoolerSerialNum); + SpoolerReplayStartOperation(SerialNum serialNum, SerialNum spoolerSerialNum); }; @@ -54,8 +50,7 @@ public: * @param serialNum the current serial number of the transaction log. * @param spoolerSerialNum the serial number of the last entry of the spooler log replay. */ - SpoolerReplayCompleteOperation(SerialNum serialNum, - SerialNum spoolerSerialNum); + SpoolerReplayCompleteOperation(SerialNum serialNum, SerialNum spoolerSerialNum); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp index d31f1faec77..dc83848152c 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp @@ -51,12 +51,11 @@ UpdateOperation::serializeUpdate(vespalib::nbostream &os) const } void -UpdateOperation::deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) +UpdateOperation::deserializeUpdate(vespalib::nbostream && is, const document::DocumentTypeRepo &repo) { - document::ByteBuffer buf(is.peek(), is.size()); - DocumentUpdate::UP update = (getType() == UPDATE_42) ? DocumentUpdate::create42(repo, buf) : DocumentUpdate::createHEAD(repo, buf); - is.adjustReadPos(buf.getPos()); - _upd = std::move(update); + _upd = (getType() == UPDATE_42) + ? DocumentUpdate::create42(repo, is) + : DocumentUpdate::createHEAD(repo, std::move(is)); } void @@ -73,7 +72,7 @@ UpdateOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &re { DocumentOperation::deserialize(is, repo); try { - deserializeUpdate(is, repo); + deserializeUpdate(std::move(is), repo); } catch (document::DocumentTypeNotFoundException &e) { LOG(warning, "Failed deserialize update operation using unknown document type '%s'", e.getDocumentTypeName().c_str()); @@ -83,18 +82,19 @@ UpdateOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &re } void -UpdateOperation::deserializeUpdate(const DocumentTypeRepo &repo) +UpdateOperation::verifyUpdate(const DocumentTypeRepo &repo) { vespalib::nbostream stream; serializeUpdate(stream); - deserializeUpdate(stream, repo); + deserializeUpdate(std::move(stream), repo); + _upd->eagerDeserialize(); // Will trigger exceptions if incompatible } -vespalib::string UpdateOperation::toString() const { +vespalib::string +UpdateOperation::toString() const { return make_string("%s(%s, %s)", ((getType() == FeedOperation::UPDATE_42) ? "Update42" : "Update"), - _upd.get() ? - _upd->getId().getScheme().toString().c_str() : "NULL", + _upd.get() ? _upd->getId().getScheme().toString().c_str() : "NULL", docArgsToString().c_str()); } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h index 99dcbfbce6c..83e87fea096 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h @@ -19,7 +19,7 @@ private: const storage::spi::Timestamp ×tamp, const DocumentUpdateSP &upd); void serializeUpdate(vespalib::nbostream &os) const; - void deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo); + void deserializeUpdate(vespalib::nbostream && is, const document::DocumentTypeRepo &repo); public: UpdateOperation(); UpdateOperation(Type type); @@ -30,7 +30,7 @@ public: const DocumentUpdateSP &getUpdate() const { return _upd; } void serialize(vespalib::nbostream &os) const override; void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; - void deserializeUpdate(const document::DocumentTypeRepo &repo); + void verifyUpdate(const document::DocumentTypeRepo &repo); vespalib::string toString() const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp index d7f38a0cc52..c5e6acd80ed 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "wipehistoryoperation.h" +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/stringfmt.h> using vespalib::make_string; @@ -9,29 +10,29 @@ namespace proton { WipeHistoryOperation::WipeHistoryOperation() : FeedOperation(FeedOperation::WIPE_HISTORY), - _wipeTimeLimit(0) { + _wipeTimeLimit(0) +{ } WipeHistoryOperation::WipeHistoryOperation(SerialNum serialNum, fastos::TimeStamp wipeTimeLimit) : FeedOperation(FeedOperation::WIPE_HISTORY), - _wipeTimeLimit(wipeTimeLimit) { + _wipeTimeLimit(wipeTimeLimit) +{ setSerialNum(serialNum); } void WipeHistoryOperation::serialize(vespalib::nbostream &str) const { str << _wipeTimeLimit; } -void WipeHistoryOperation::deserialize(vespalib::nbostream &str, - const document::DocumentTypeRepo &) { +void WipeHistoryOperation::deserialize(vespalib::nbostream &str, const document::DocumentTypeRepo &) { fastos::TimeStamp::TimeT t; str >> t; _wipeTimeLimit = t; } vespalib::string WipeHistoryOperation::toString() const { - return make_string("WipeHistory(wipeTimeLimit=%" PRIu64 - ", serialNum=%" PRIu64 ")", + return make_string("WipeHistory(wipeTimeLimit=%" PRIu64 ", serialNum=%" PRIu64 ")", _wipeTimeLimit.ns(), getSerialNum()); } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.h index 80b551dbd92..0cf7256bf27 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.h @@ -12,14 +12,13 @@ class WipeHistoryOperation : public FeedOperation { public: WipeHistoryOperation(); WipeHistoryOperation(SerialNum serialNum, fastos::TimeStamp wipeTimeLimit); - virtual ~WipeHistoryOperation() {} + ~WipeHistoryOperation() override {} fastos::TimeStamp getWipeTimeLimit() const { return _wipeTimeLimit; } - virtual void serialize(vespalib::nbostream &str) const override; - virtual void deserialize(vespalib::nbostream &str, - const document::DocumentTypeRepo &) override; - virtual vespalib::string toString() const override; + void serialize(vespalib::nbostream &str) const override; + void deserialize(vespalib::nbostream &str, const document::DocumentTypeRepo &) override; + vespalib::string toString() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index bab3494ca5c..6475efdaabb 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -6,7 +6,9 @@ #include <vespa/metrics/loadmetric.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/document/update/documentupdate.h> +#include <vespa/document/base/exceptions.h> #include <vespa/log/log.h> @@ -25,6 +27,7 @@ using storage::spi::Result; using vespalib::IllegalStateException; using vespalib::Sequence; using vespalib::make_string; +using std::make_unique; using namespace std::chrono_literals; @@ -377,6 +380,18 @@ PersistenceEngine::update(const Bucket& b, Timestamp t, const DocumentUpdate::SP upd->getId().toString().c_str(), state.message().c_str())); } } + try { + upd->eagerDeserialize(); + } catch (document::FieldNotFoundException & e) { + return UpdateResult(Result::TRANSIENT_ERROR, + make_string("Update operation rejected for document '%s' of type '%s': 'Field not found'", + upd->getId().toString().c_str(), upd->getType().getName().c_str())); + } catch (document::DocumentTypeNotFoundException & e) { + return UpdateResult(Result::TRANSIENT_ERROR, + make_string("Update operation rejected for document '%s' of type '%s'.", + upd->getId().toString().c_str(), e.getDocumentTypeName().c_str())); + + } std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); DocTypeName docType(upd->getType()); LOG(spam, "update(%s, %" PRIu64 ", (\"%s\", \"%s\"), createIfNonExistent='%s')", diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index f2ab35e0ccf..a6c696d08fb 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -87,37 +87,36 @@ public: PersistenceEngine(IPersistenceEngineOwner &owner, const IResourceWriteFilter &writeFilter, ssize_t defaultSerializedSize, bool ignoreMaxBytes); - ~PersistenceEngine(); + ~PersistenceEngine() override; IPersistenceHandler::SP putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType, const IPersistenceHandler::SP &handler); IPersistenceHandler::SP removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType); // Implements PersistenceProvider - virtual Result initialize() override; - virtual PartitionStateListResult getPartitionStates() const override; - virtual BucketIdListResult listBuckets(BucketSpace bucketSpace, PartitionId) const override; - virtual Result setClusterState(BucketSpace bucketSpace, const ClusterState& calc) override; - virtual Result setActiveState(const Bucket& bucket, BucketInfo::ActiveState newState) override; - virtual BucketInfoResult getBucketInfo(const Bucket&) const override; - virtual Result put(const Bucket&, Timestamp, const std::shared_ptr<document::Document>&, Context&) override; - virtual RemoveResult remove(const Bucket&, Timestamp, const document::DocumentId&, Context&) override; - virtual UpdateResult update(const Bucket&, Timestamp, - const std::shared_ptr<document::DocumentUpdate>&, Context&) override; - virtual GetResult get(const Bucket&, const document::FieldSet&, - const document::DocumentId&, Context&) const override; - virtual CreateIteratorResult createIterator(const Bucket&, const document::FieldSet&, const Selection&, - IncludedVersions, Context&) override; - virtual IterateResult iterate(IteratorId, uint64_t maxByteSize, Context&) const override; - virtual Result destroyIterator(IteratorId, Context&) override; - - virtual Result createBucket(const Bucket &bucketId, Context &) override ; - virtual Result deleteBucket(const Bucket&, Context&) override; - virtual BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; - virtual Result split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&) override; - virtual Result join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&) override; - - virtual Result maintain(const Bucket&, MaintenanceLevel) override; + Result initialize() override; + PartitionStateListResult getPartitionStates() const override; + BucketIdListResult listBuckets(BucketSpace bucketSpace, PartitionId) const override; + Result setClusterState(BucketSpace bucketSpace, const ClusterState& calc) override; + Result setActiveState(const Bucket& bucket, BucketInfo::ActiveState newState) override; + BucketInfoResult getBucketInfo(const Bucket&) const override; + Result put(const Bucket&, Timestamp, const std::shared_ptr<document::Document>&, Context&) override; + RemoveResult remove(const Bucket&, Timestamp, const document::DocumentId&, Context&) override; + UpdateResult update(const Bucket&, Timestamp, + const std::shared_ptr<document::DocumentUpdate>&, Context&) override; + GetResult get(const Bucket&, const document::FieldSet&, const document::DocumentId&, Context&) const override; + CreateIteratorResult createIterator(const Bucket&, const document::FieldSet&, const Selection&, + IncludedVersions, Context&) override; + IterateResult iterate(IteratorId, uint64_t maxByteSize, Context&) const override; + Result destroyIterator(IteratorId, Context&) override; + + Result createBucket(const Bucket &bucketId, Context &) override ; + Result deleteBucket(const Bucket&, Context&) override; + BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; + Result split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&) override; + Result join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&) override; + + Result maintain(const Bucket&, MaintenanceLevel) override; void destroyIterators(); void propagateSavedClusterState(BucketSpace bucketSpace, IPersistenceHandler &handler); diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp index da03adb3fe4..46096fead05 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp @@ -12,7 +12,7 @@ GidToLidChangeListener::GidToLidChangeListener(search::ISequencedTaskExecutor &a const vespalib::string &name, const vespalib::string &docTypeName) : _attributeFieldWriter(attributeFieldWriter), - _executorId(_attributeFieldWriter.getExecutorId(attr->getName())), + _executorId(_attributeFieldWriter.getExecutorId(attr->getNamePrefix())), _attr(std::move(attr)), _refCount(refCount), _name(name), diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp index 711f8ebdcaf..1df812b0c61 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp @@ -38,6 +38,9 @@ bool fastPartialUpdateAttribute(BasicType::Type attrType) { (attrType != BasicType::Type::REFERENCE)); } +bool isStructFieldAttribute(const vespalib::string &name) { + return name.find('.') != vespalib::string::npos; +} FilterAttributeManager::AttributeSet getAttributeSetToPopulate(const ARIConfig &newCfg, @@ -100,7 +103,8 @@ getFieldsToPopulate(const ARIConfig &newCfg, // keep the original in order to preserve annotations. bool wasStringIndexField = oldIndexschemaInspector.isStringIndex(name); bool populateField = !inNewAttrMgr && unchangedField && !wasStringIndexField && - fastPartialUpdateAttribute(attrType.type()); + fastPartialUpdateAttribute(attrType.type()) && + !isStructFieldAttribute(name); LOG(debug, "getFieldsToPopulate(): name='%s', inNewAttrMgr=%s, unchangedField=%s, " "wasStringIndexField=%s, dataType=%s, populate=%s", name.c_str(), toStr(inNewAttrMgr), toStr(unchangedField), diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp index 87735dfef5d..6a000973165 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp @@ -3,7 +3,7 @@ #include "combiningfeedview.h" #include <vespa/document/fieldvalue/document.h> #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> -#include <vespa/searchcore/proton/feedoperation/moveoperation.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/log/log.h> @@ -13,17 +13,15 @@ using document::DocumentTypeRepo; using document::DocumentId; using search::IDestructorCallback; -namespace proton -{ +namespace proton { -namespace -{ +namespace { std::shared_ptr<const DocumentTypeRepo> getRepo(const std::vector<IFeedView::SP> &views) { for (const auto &view : views) { - if (view.get() == NULL) + if (view.get() == nullptr) continue; return view->getDocumentTypeRepo(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h index 6644f21a3b7..3546fdbea71 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h @@ -3,21 +3,14 @@ #pragma once #include "ifeedview.h" -#include <vespa/searchcore/proton/common/feedtoken.h> -#include <vespa/searchcore/proton/feedoperation/deletebucketoperation.h> -#include <vespa/searchcore/proton/feedoperation/joinbucketsoperation.h> -#include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h> -#include <vespa/searchcore/proton/feedoperation/putoperation.h> -#include <vespa/searchcore/proton/feedoperation/removeoperation.h> -#include <vespa/searchcore/proton/feedoperation/splitbucketoperation.h> -#include <vespa/searchcore/proton/feedoperation/updateoperation.h> -#include <vespa/searchcore/proton/feedoperation/createbucketoperation.h> -#include <vespa/searchlib/common/serialnum.h> #include "replaypacketdispatcher.h" #include "ibucketstatecalculator.h" +#include <vespa/searchcore/proton/common/feedtoken.h> +#include <vespa/searchlib/common/serialnum.h> namespace proton { +class DocumentOperation; class CombiningFeedView : public IFeedView { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 61ce0e6dc6f..ad7a2d57c89 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -17,6 +17,7 @@ #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchcore/proton/common/statusreport.h> +#include <vespa/searchcore/proton/feedoperation/noopoperation.h> #include <vespa/searchcore/proton/index/index_writer.h> #include <vespa/searchcore/proton/initializer/task_runner.h> #include <vespa/searchcore/proton/metrics/attribute_metrics_collection.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp index 6364e772f94..1c2406b2acf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp @@ -5,6 +5,7 @@ #include "operationdonecontext.h" #include "removedonecontext.h" #include "putdonecontext.h" +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> using document::Document; @@ -71,7 +72,7 @@ FastAccessFeedView::FastAccessFeedView(const StoreOnlyFeedView::Context &storeOn _docIdLimit(ctx._docIdLimit) {} -FastAccessFeedView::~FastAccessFeedView() {} +FastAccessFeedView::~FastAccessFeedView() = default; void FastAccessFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op) diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 66e721f9222..97cc25e635f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -14,6 +14,7 @@ #include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/searchcore/proton/persistenceengine/transport_latch.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/searchlib/common/gatecallback.h> @@ -522,7 +523,7 @@ FeedHandler::considerUpdateOperationForRejection(FeedToken &token, UpdateOperati */ if (_documentType != &update.getType()) { try { - op.deserializeUpdate(*_repo); + op.verifyUpdate(*_repo); } catch (document::FieldNotFoundException &e) { if (token) { auto message = make_string("Update operation rejected for document '%s' of type '%s': 'Field not found'", diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstate.h b/searchcore/src/vespa/searchcore/proton/server/feedstate.h index 472f5cb224f..fa0a1702499 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstate.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedstate.h @@ -25,6 +25,7 @@ private: Type _type; protected: + using FeedOperationUP = std::unique_ptr<FeedOperation>; void throwExceptionInReceive(const vespalib::string &docType, uint64_t serialRangeFrom, uint64_t serialRangeTo, size_t packetSize); void throwExceptionInHandleOperation(const vespalib::string &docType, const FeedOperation &op); @@ -38,7 +39,7 @@ public: Type getType() const { return _type; } vespalib::string getName() const; - virtual void handleOperation(FeedToken token, std::unique_ptr<FeedOperation> op) = 0; + virtual void handleOperation(FeedToken token, FeedOperationUP op) = 0; virtual void receive(const PacketWrapper::SP &wrap, vespalib::Executor &executor) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp index f0866347f59..ae323bc93df 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp @@ -6,6 +6,7 @@ #include "ireplayconfig.h" #include "replaypacketdispatcher.h" #include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/vespalib/util/closuretask.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.h b/searchcore/src/vespa/searchcore/proton/server/feedstates.h index 963a78d0d6b..a9224669d87 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.h @@ -28,16 +28,13 @@ public: { } - virtual void handleOperation(FeedToken, FeedOperation::UP op) override { + void handleOperation(FeedToken, FeedOperationUP op) override { throwExceptionInHandleOperation(_doc_type_name, *op); } - virtual void - receive(const PacketWrapper::SP &wrap, vespalib::Executor &) override { - throwExceptionInReceive(_doc_type_name.c_str(), - wrap->packet.range().from(), - wrap->packet.range().to(), - wrap->packet.size()); + void receive(const PacketWrapper::SP &wrap, vespalib::Executor &) override { + throwExceptionInReceive(_doc_type_name.c_str(), wrap->packet.range().from(), + wrap->packet.range().to(), wrap->packet.size()); } }; @@ -57,12 +54,11 @@ public: IReplayConfig &replay_config, FeedConfigStore &config_store); - virtual void handleOperation(FeedToken, FeedOperation::UP op) override { + void handleOperation(FeedToken, FeedOperationUP op) override { throwExceptionInHandleOperation(_doc_type_name, *op); } - virtual void receive(const PacketWrapper::SP &wrap, - vespalib::Executor &executor) override; + void receive(const PacketWrapper::SP &wrap, vespalib::Executor &executor) override; }; @@ -79,15 +75,13 @@ public: _handler(handler) { } - void handleOperation(FeedToken token, FeedOperation::UP op) override { + void handleOperation(FeedToken token, FeedOperationUP op) override { _handler.performOperation(std::move(token), std::move(op)); } void receive(const PacketWrapper::SP &wrap, vespalib::Executor &) override { - throwExceptionInReceive(_handler.getDocTypeName().c_str(), - wrap->packet.range().from(), - wrap->packet.range().to(), - wrap->packet.size()); + throwExceptionInReceive(_handler.getDocTypeName().c_str(), wrap->packet.range().from(), + wrap->packet.range().to(), wrap->packet.size()); } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h b/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h index 210ece3a36a..e93821e2a36 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h @@ -2,10 +2,28 @@ #pragma once -#include <vespa/searchcore/proton/feedoperation/operations.h> +namespace document { class DocumentTypeRepo; } namespace proton { +class PutOperation; +class RemoveOperation; +class UpdateOperation; +class NoopOperation; +class NewConfigOperation; +class WipeHistoryOperation; +class DeleteBucketOperation; +class SplitBucketOperation; +class JoinBucketsOperation; +class PruneRemovedDocumentsOperation; +class SpoolerReplayStartOperation; +class SpoolerReplayCompleteOperation; +class MoveOperation; +class CreateBucketOperation; +class CompactLidSpaceOperation; + +namespace feedoperation { class IStreamHandler; } + /** * Interface used to handle the various feed operations during * replay of the transaction log. @@ -29,7 +47,7 @@ struct IReplayPacketHandler virtual void replay(const CreateBucketOperation &op) = 0; virtual void replay(const CompactLidSpaceOperation &op) = 0; - virtual NewConfigOperation::IStreamHandler &getNewConfigStreamHandler() = 0; + virtual feedoperation::IStreamHandler &getNewConfigStreamHandler() = 0; virtual const document::DocumentTypeRepo &getDeserializeRepo() = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp index cdb42906fc2..42451f08315 100644 --- a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "replaypacketdispatcher.h" +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/util/serializableexceptions.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.h b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.h index fc6f99471d8..771a79ddad9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.h @@ -7,6 +7,7 @@ namespace proton { +class FeedOperation; /** * Utility class that deserializes packet entries into feed operations * during replay from the transaction log and dispatches the feed operations @@ -22,14 +23,11 @@ private: void replay(OperationType &op, vespalib::nbostream &is, const Packet::Entry &entry); protected: - virtual void - store(const FeedOperation &op); + virtual void store(const FeedOperation &op); public: ReplayPacketDispatcher(IReplayPacketHandler &handler); - - virtual - ~ReplayPacketDispatcher(); + virtual ~ReplayPacketDispatcher(); void replayEntry(const Packet::Entry &entry); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 29615b0daf9..22ff1b90de4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -13,6 +13,7 @@ #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> #include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/document/datatype/documenttype.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 825e14b4368..37f67399f4d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -13,7 +13,7 @@ #include <vespa/searchcore/proton/common/feeddebugger.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> -#include <vespa/searchcore/proton/feedoperation/feedoperation.h> +#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h> #include <vespa/searchcore/proton/persistenceengine/resulthandler.h> #include <vespa/searchcore/proton/reference/pending_notify_remove_done.h> #include <vespa/searchcorespi/index/ithreadingservice.h> @@ -23,7 +23,6 @@ namespace search { class IDestructorCallback; } -namespace document { class GLobalId; } namespace proton { @@ -35,6 +34,8 @@ class RemoveDoneContext; class CommitTimeTracker; class IGidToLidChangeHandler; class IFieldUpdateCallback; +class RemoveDocumentsOperation; +class DocumentOperation; namespace documentmetastore { class ILidReuseDelayer; } diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 1ba264e0bfe..b911284a1b4 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -19,6 +19,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/document/annotation/spanlist.h> #include <vespa/document/annotation/spantree.h> #include <vespa/searchlib/util/rand48.h> diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 5fcc49fc228..5148ab2be34 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -32,8 +32,7 @@ using search::index::DummyFileHeaderContext; using search::attribute::BasicType; using search::attribute::IAttributeVector; -namespace -{ +namespace { vespalib::string empty; vespalib::string tmpDir("tmp"); @@ -2315,6 +2314,23 @@ AttributeTest::testPendingCompaction() populateSimple(iv, 1, 2); // should not trigger new compaction } +void testNamePrefix() { + Config cfg(BasicType::INT32, CollectionType::SINGLE); + AttributeVector::SP vFlat = createAttribute("sfsint32_pc", cfg); + AttributeVector::SP vS1 = createAttribute("sfsint32_pc.abc", cfg); + AttributeVector::SP vS2 = createAttribute("sfsint32_pc.xyz", cfg); + AttributeVector::SP vSS1 = createAttribute("sfsint32_pc.xyz.abc", cfg); + EXPECT_EQUAL("sfsint32_pc", vFlat->getName()); + EXPECT_EQUAL("sfsint32_pc", vFlat->getNamePrefix()); + EXPECT_EQUAL("sfsint32_pc.abc", vS1->getName()); + EXPECT_EQUAL("sfsint32_pc", vS1->getNamePrefix()); + EXPECT_EQUAL("sfsint32_pc.xyz", vS2->getName()); + EXPECT_EQUAL("sfsint32_pc", vS2->getNamePrefix()); + EXPECT_EQUAL("sfsint32_pc.xyz.abc", vSS1->getName()); + EXPECT_EQUAL("sfsint32_pc", vSS1->getNamePrefix()); + +} + void deleteDataDirs() { @@ -2361,6 +2377,7 @@ int AttributeTest::Main() TEST_DO(requireThatAddressSpaceUsageIsReported()); testReaderDuringLastUpdate(); TEST_DO(testPendingCompaction()); + TEST_DO(testNamePrefix()); deleteDataDirs(); TEST_DONE(); diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index 805a6d3b962..f8ab03d7710 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp @@ -122,8 +122,8 @@ TEST_F("require that task with same string component id are serialized", Fixture std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); EXPECT_EQUAL(0, tv->_val); auto test2 = [=]() { tv->modify(14, 42); }; - f._threads.execute("0", [=]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute("0", test2); + f._threads.execute(f._threads.getExecutorId("0"), [=]() { usleep(2000); tv->modify(0, 14); }); + f._threads.execute(f._threads.getExecutorId("0"), test2); tv->wait(2); EXPECT_EQUAL(0, tv->_fail); EXPECT_EQUAL(42, tv->_val); @@ -132,8 +132,7 @@ TEST_F("require that task with same string component id are serialized", Fixture EXPECT_EQUAL(42, tv->_val); } -namespace -{ +namespace { int detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int tryLimit) { @@ -141,8 +140,8 @@ int detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int t for (tryCnt = 0; tryCnt < tryLimit; ++tryCnt) { std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); EXPECT_EQUAL(0, tv->_val); - f._threads.execute("0", [=]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute(altComponentId, [=]() { tv->modify(14, 42); }); + f._threads.execute(f._threads.getExecutorId("0"), [=]() { usleep(2000); tv->modify(0, 14); }); + f._threads.execute(f._threads.getExecutorId(altComponentId), [=]() { tv->modify(14, 42); }); tv->wait(2); if (tv->_fail != 1) { continue; diff --git a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp index 1da912ccb3a..0cf13443142 100644 --- a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp +++ b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp @@ -16,7 +16,8 @@ using namespace search::fef::test; using namespace search::features; using CollectionType = FieldInfo::CollectionType; -const std::string featureName("nativeDotProduct(foo)"); +const std::string fooFeatureName("nativeDotProduct(foo)"); +const std::string anyFeatureName("nativeDotProduct"); struct BlueprintFactoryFixture { BlueprintFactory factory; @@ -78,7 +79,8 @@ struct RankFixture : BlueprintFactoryFixture, IndexFixture { std::vector<TermFieldHandle> fooHandles; std::vector<TermFieldHandle> barHandles; RankFixture(const std::vector<uint32_t> &fooWeights, - const std::vector<uint32_t> &barWeights) + const std::vector<uint32_t> &barWeights, + const vespalib::string &featureName = fooFeatureName) : queryEnv(&indexEnv), rankSetup(factory, indexEnv), mdl(), match_data(), rankProgram(), fooHandles(), barHandles() { @@ -152,6 +154,12 @@ TEST_FF("require that setup fails for unknown field", NativeDotProductBlueprint, EXPECT_TRUE(!((Blueprint&)f1).setup(f2.indexEnv, std::vector<vespalib::string>(1, "unknown"))); } +TEST_FF("require that setup can be done without field", NativeDotProductBlueprint, IndexFixture) { + DummyDependencyHandler deps(f1); + f1.setName(vespalib::make_string("%s", f1.getBaseName().c_str())); + EXPECT_TRUE(((Blueprint&)f1).setup(f2.indexEnv, std::vector<vespalib::string>())); +} + TEST_F("require that not searching a field will give it 0.0 dot product", RankFixture(vec(), vec(1, 2, 3))) { EXPECT_EQUAL(0.0, f1.getScore(10)); } @@ -183,11 +191,18 @@ TEST_F("require that data from other fields is ignored", RankFixture(vec(1, 3), EXPECT_EQUAL(14, f1.getScore(10)); } +TEST_F("require that not specifying field includes all term/field combinations", RankFixture(vec(1, 3), vec(5, 7), anyFeatureName)) { + f1.setFooWeight(0, 10, 2); + f1.setFooWeight(1, 10, 4); + f1.setBarWeight(0, 10, 6); + f1.setBarWeight(1, 10, 8); + EXPECT_EQUAL(100, f1.getScore(10)); +} + TEST_F("require that negative weights in the index works", RankFixture(vec(1, 3), vec())) { f1.setFooWeight(0, 10, 2); f1.setFooWeight(1, 10, -4); EXPECT_EQUAL(-10, f1.getScore(10)); } - TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 345c66ec672..0e57f520673 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -32,6 +32,7 @@ LOG_SETUP("prod_features_test"); #include <vespa/searchlib/features/querytermcountfeature.h> #include <vespa/searchlib/features/randomfeature.h> #include <vespa/searchlib/features/random_normal_feature.h> +#include <vespa/searchlib/features/random_normal_stable_feature.h> #include <vespa/searchlib/features/rankingexpressionfeature.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/features/termfeature.h> @@ -105,6 +106,7 @@ Test::Main() TEST_DO(testQueryTermCount()); TEST_FLUSH(); TEST_DO(testRandom()); TEST_FLUSH(); TEST_DO(testRandomNormal()); TEST_FLUSH(); + TEST_DO(testRandomNormalStable()); TEST_FLUSH(); TEST_DO(testRankingExpression()); TEST_FLUSH(); TEST_DO(testTerm()); TEST_FLUSH(); TEST_DO(testTermDistance()); TEST_FLUSH(); @@ -1727,17 +1729,16 @@ Test::testRandom() } void -Test::testRandomNormal() -{ +Test::testRandomNormal() { { // Test blueprint. RandomNormalBlueprint pt; EXPECT_TRUE(assertCreateInstance(pt, "randomNormal")); StringList params, in, out; - FT_SETUP_OK (pt, params, in, out.add("out")); - FT_SETUP_OK (pt, params.add("0.5").add("1.0"), in, out); - FT_SETUP_OK (pt, params.add("val1"), in, out); + FT_SETUP_OK(pt, params, in, out.add("out")); + FT_SETUP_OK(pt, params.add("0.5").add("1.0"), in, out); + FT_SETUP_OK(pt, params.add("val1"), in, out); FT_DUMP_EMPTY(_factory, "randomNormal"); } @@ -1766,7 +1767,50 @@ Test::testRandomNormal() for (uint32_t i = 0; i < 5; ++i) { rr.clear(); ASSERT_TRUE(ft1.executeOnly(rr, i + 1)); - ASSERT_TRUE(ft2.execute(((rr.getScore("randomNormal(0.0,0.1)")-0.0)/0.1) * 0.2 + 1.0, EPS, i + 1)); + ASSERT_TRUE(ft2.execute(((rr.getScore("randomNormal(0.0,0.1)") - 0.0) / 0.1) * 0.2 + 1.0, EPS, i + 1)); + } + } +} + +void +Test::testRandomNormalStable() { + { // Test blueprint. + RandomNormalStableBlueprint pt; + + EXPECT_TRUE(assertCreateInstance(pt, "randomNormalStable")); + + StringList params, in, out; + FT_SETUP_OK(pt, params, in, out.add("out")); + FT_SETUP_OK(pt, params.add("0.5").add("1.0"), in, out); + FT_SETUP_OK(pt, params.add("val1"), in, out); + + FT_DUMP_EMPTY(_factory, "randomNormalStable"); + } + + { // Test setting of mean and stddev values, and seed + FtFeatureTest ft1(_factory, "randomNormalStable(0.0,0.1)"); + FtFeatureTest ft2(_factory, "randomNormalStable(1.0,0.2)"); + ft1.getIndexEnv().getProperties().add("randomNormalStable(0.0,0.1).seed", "100"); + ft2.getIndexEnv().getProperties().add("randomNormalStable(1.0,0.2).seed", "100"); + ASSERT_TRUE(ft1.setup()); + ASSERT_TRUE(ft2.setup()); + RankResult rr; + for (uint32_t i = 0; i < 5; ++i) { + rr.clear(); + ASSERT_TRUE(ft1.executeOnly(rr, i + 1)); + ASSERT_TRUE(ft2.execute(((rr.getScore("randomNormalStable(0.0,0.1)") - 0.0) / 0.1) * 0.2 + 1.0, EPS, i + 1)); + } + } + { // Test executor (randomNormalStable) + FtFeatureTest ft1(_factory, "randomNormalStable"); + FtFeatureTest ft2(_factory, "randomNormalStable"); + ASSERT_TRUE(ft1.setup()); + ASSERT_TRUE(ft2.setup()); + RankResult rr; + for (uint32_t i = 0; i < 5; ++i) { + rr.clear(); + ASSERT_TRUE(ft1.executeOnly(rr, i + 1)); + ASSERT_TRUE(ft2.execute(rr.getScore("randomNormalStable"), EPS, i + 1)); } } } diff --git a/searchlib/src/tests/features/prod_features.h b/searchlib/src/tests/features/prod_features.h index 0d234ca674e..d7bf001bedf 100644 --- a/searchlib/src/tests/features/prod_features.h +++ b/searchlib/src/tests/features/prod_features.h @@ -35,6 +35,7 @@ public: void testQueryTermCount(); void testRandom(); void testRandomNormal(); + void testRandomNormalStable(); void testRankingExpression(); void testTerm(); void testTermDistance(); diff --git a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp index d89883bc417..45ebdd78fb3 100644 --- a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp +++ b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp @@ -5,17 +5,24 @@ #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/queryeval/same_element_blueprint.h> +#include <vespa/searchlib/queryeval/same_element_search.h> +#include <vespa/searchlib/queryeval/emptysearch.h> +#include <vespa/searchcommon/attribute/i_search_context.h> +#include <vespa/searchlib/attribute/elementiterator.h> using namespace search::fef; using namespace search::queryeval; +using search::attribute::ElementIterator; -std::unique_ptr<SameElementBlueprint> make_blueprint(const std::vector<FakeResult> &children) { +std::unique_ptr<SameElementBlueprint> make_blueprint(const std::vector<FakeResult> &children, bool fake_attr = false) { auto result = std::make_unique<SameElementBlueprint>(); for (size_t i = 0; i < children.size(); ++i) { uint32_t field_id = i; vespalib::string field_name = vespalib::make_string("f%u", field_id); FieldSpec field = result->getNextChildField(field_name, field_id); - result->addTerm(std::make_unique<FakeBlueprint>(field, children[i])); + auto fake = std::make_unique<FakeBlueprint>(field, children[i]); + fake->is_attr(fake_attr); + result->addTerm(std::move(fake)); } return result; } @@ -96,4 +103,17 @@ TEST("require that children are sorted") { EXPECT_EQUAL(dynamic_cast<SameElementBlueprint&>(*bp).terms()[2]->getState().estimate().estHits, 4u); } +TEST("require that attribute iterators are wrapped for element unpacking") { + auto a = make_result({{5, {1,3,7}}}); + auto b = make_result({{5, {3,5,10}}}); + auto bp = finalize(make_blueprint({a,b}, true), true); + auto md = MatchData::makeTestInstance(0, 0); + auto search = bp->createSearch(*md, false); + SameElementSearch *se = dynamic_cast<SameElementSearch*>(search.get()); + ASSERT_TRUE(se != nullptr); + ASSERT_EQUAL(se->children().size(), 2u); + EXPECT_TRUE(dynamic_cast<ElementIterator*>(se->children()[0].get()) != nullptr); + EXPECT_TRUE(dynamic_cast<ElementIterator*>(se->children()[1].get()) != nullptr); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h b/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h index 05347e790fb..a8b2a722c01 100644 --- a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h +++ b/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h @@ -89,14 +89,14 @@ public: * call sync before tearing down pointed to/referenced data. * All tasks must be scheduled from same thread. * - * @param componentId component id - * @param function function to be wrapped in a task and later executed + * @param id executor id + * @param function function to be wrapped in a task and later executed */ template <class FunctionType> - void execute(vespalib::stringref componentId, FunctionType &&function) { - ExecutorId id = getExecutorId(componentId); + void execute(ExecutorId id, FunctionType &&function) { executeTask(id, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); } + }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index bd847fe35b5..16401a67424 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -44,6 +44,7 @@ vespa_add_library(searchlib_features OBJECT queryterm.cpp querytermcountfeature.cpp random_normal_feature.cpp + random_normal_stable_feature.cpp randomfeature.cpp rankingexpressionfeature.cpp raw_score_feature.cpp diff --git a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp index cb8136e8b7f..7865e32849f 100644 --- a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp @@ -8,6 +8,23 @@ using namespace search::fef; namespace search { namespace features { +NativeDotProductExecutor::NativeDotProductExecutor(const search::fef::IQueryEnvironment &env) + : FeatureExecutor(), + _pairs(), + _md(nullptr) +{ + for (uint32_t i = 0; i < env.getNumTerms(); ++i) { + const search::fef::ITermData *td = env.getTerm(i); + auto weight = td->getWeight(); + for (size_t f = 0; f < td->numFields(); ++f) { + auto handle = td->field(f).getHandle(); + if (handle != search::fef::IllegalHandle) { + _pairs.emplace_back(handle, weight); + } + } + } +} + NativeDotProductExecutor::NativeDotProductExecutor(const search::fef::IQueryEnvironment &env, uint32_t fieldId) : FeatureExecutor(), _pairs(), @@ -46,15 +63,21 @@ bool NativeDotProductBlueprint::setup(const IIndexEnvironment &, const ParameterList ¶ms) { - _field = params[0].asField(); - describeOutput("out", "dot product between query term weights and match weights for the given field"); + if (params.size() > 0) { + _field = params[0].asField(); + } + describeOutput("out", "dot product between query term weights and match weights"); return true; } FeatureExecutor & NativeDotProductBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash &stash) const { - return stash.create<NativeDotProductExecutor>(queryEnv, _field->id()); + if (_field) { + return stash.create<NativeDotProductExecutor>(queryEnv, _field->id()); + } else { + return stash.create<NativeDotProductExecutor>(queryEnv); + } } } // namespace features diff --git a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.h b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.h index a71d23f3158..33c5c89c88b 100644 --- a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.h +++ b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.h @@ -20,6 +20,7 @@ private: void handle_bind_match_data(const fef::MatchData &md) override; public: + NativeDotProductExecutor(const fef::IQueryEnvironment &env); NativeDotProductExecutor(const fef::IQueryEnvironment &env, uint32_t fieldId); void execute(uint32_t docId) override; }; @@ -31,13 +32,13 @@ class NativeDotProductBlueprint : public fef::Blueprint private: const fef::FieldInfo *_field; public: - NativeDotProductBlueprint() : Blueprint("nativeDotProduct"), _field(0) {} + NativeDotProductBlueprint() : Blueprint("nativeDotProduct"), _field(nullptr) {} void visitDumpFeatures(const fef::IIndexEnvironment &, fef::IDumpFeatureVisitor &) const override {} fef::Blueprint::UP createInstance() const override { return Blueprint::UP(new NativeDotProductBlueprint()); } fef::ParameterDescriptions getDescriptions() const override { - return fef::ParameterDescriptions().desc().field(); + return fef::ParameterDescriptions().desc().field().desc(); } bool setup(const fef::IIndexEnvironment &env, const fef::ParameterList ¶ms) override; fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp index 67f30c4eb93..c83ec80f6b5 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp @@ -4,7 +4,6 @@ #include "utils.h" #include <vespa/searchlib/fef/properties.h> #include <vespa/fastos/time.h> -#include <cmath> #include <vespa/log/log.h> LOG_SETUP(".features.randomnormalfeature"); @@ -14,46 +13,18 @@ namespace features { RandomNormalExecutor::RandomNormalExecutor(uint64_t seed, double mean, double stddev) : search::fef::FeatureExecutor(), - _rnd(), - _mean(mean), - _stddev(stddev), - _hasSpare(false), - _spare(0.0) - + _rnd(mean, stddev, true) { LOG(debug, "RandomNormalExecutor: seed=%zu, mean=%f, stddev=%f", seed, mean, stddev); - _rnd.srand48(seed); + _rnd.seed(seed); } -/** - * Draws a random number from the Gaussian distribution - * using the Marsaglia polar method. - */ void RandomNormalExecutor::execute(uint32_t) { - feature_t result = _spare; - if (_hasSpare) { - _hasSpare = false; - } else { - _hasSpare = true; - - feature_t u, v, s; - do { - u = (_rnd.lrand48() / (feature_t)0x80000000u) * 2.0 - 1.0; - v = (_rnd.lrand48() / (feature_t)0x80000000u) * 2.0 - 1.0; - s = u * u + v * v; - } while ( (s >= 1.0) || (s == 0.0) ); - s = std::sqrt(-2.0 * std::log(s) / s); - - _spare = v * s; // saved for next invocation - result = u * s; - } - - outputs().set_number(0, _mean + _stddev * result); + outputs().set_number(0, _rnd.next()); } - RandomNormalBlueprint::RandomNormalBlueprint() : search::fef::Blueprint("randomNormal"), _seed(0), @@ -82,7 +53,6 @@ RandomNormalBlueprint::setup(const search::fef::IIndexEnvironment & env, if (p.found()) { _seed = util::strToNum<uint64_t>(p.get()); } - if (params.size() > 0) { _mean = params[0].asDouble(); } diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.h b/searchlib/src/vespa/searchlib/features/random_normal_feature.h index 92aa3117b91..2d2429371d9 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_feature.h +++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.h @@ -4,7 +4,7 @@ #include <vespa/searchlib/fef/blueprint.h> #include <vespa/searchlib/fef/featureexecutor.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/searchlib/util/random_normal.h> namespace search { namespace features { @@ -17,12 +17,7 @@ namespace features { **/ class RandomNormalExecutor : public fef::FeatureExecutor { private: - Rand48 _rnd; - double _mean; - double _stddev; - - bool _hasSpare; - double _spare; + RandomNormal _rnd; // seeded once per query public: RandomNormalExecutor(uint64_t seed, double mean, double stddev); diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp new file mode 100644 index 00000000000..5f3cf7fd063 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp @@ -0,0 +1,82 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "random_normal_stable_feature.h" +#include "utils.h" +#include <vespa/searchlib/fef/properties.h> +#include <vespa/fastos/time.h> + +#include <vespa/log/log.h> +LOG_SETUP(".features.randomnormalstablefeature"); + +namespace search { +namespace features { + +RandomNormalStableExecutor::RandomNormalStableExecutor(uint64_t seed, double mean, double stddev) : + search::fef::FeatureExecutor(), + _rnd(mean, stddev, false), // don't use spares, as we reset seed on every generation + _seed(seed) +{ + LOG(debug, "RandomNormalStableExecutor: seed=%zu, mean=%f, stddev=%f", seed, mean, stddev); +} + +void +RandomNormalStableExecutor::execute(uint32_t docId) +{ + _rnd.seed(_seed + docId); + outputs().set_number(0, _rnd.next()); +} + +RandomNormalStableBlueprint::RandomNormalStableBlueprint() : + search::fef::Blueprint("randomNormalStable"), + _seed(0), + _mean(0.0), + _stddev(1.0) +{ +} + +void +RandomNormalStableBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &, + search::fef::IDumpFeatureVisitor &) const +{ +} + +search::fef::Blueprint::UP +RandomNormalStableBlueprint::createInstance() const +{ + return search::fef::Blueprint::UP(new RandomNormalStableBlueprint()); +} + +bool +RandomNormalStableBlueprint::setup(const search::fef::IIndexEnvironment & env, + const search::fef::ParameterList & params) +{ + search::fef::Property p = env.getProperties().lookup(getName(), "seed"); + if (p.found()) { + _seed = util::strToNum<uint64_t>(p.get()); + } + if (params.size() > 0) { + _mean = params[0].asDouble(); + } + if (params.size() > 1) { + _stddev = params[1].asDouble(); + } + + describeOutput("out" , "A random value drawn from the Gaussian distribution that is stable for a given match (document and query)"); + + return true; +} + +search::fef::FeatureExecutor & +RandomNormalStableBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespalib::Stash &stash) const +{ + uint64_t seed = _seed; + if (seed == 0) { + seed = util::strToNum<uint64_t> + (env.getProperties().lookup(getName(), "seed").get("1024")); // default seed + } + return stash.create<RandomNormalStableExecutor>(seed, _mean, _stddev); +} + + +} // namespace features +} // namespace search diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.h b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.h new file mode 100644 index 00000000000..129c929ba3d --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.h @@ -0,0 +1,67 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/fef/blueprint.h> +#include <vespa/searchlib/fef/featureexecutor.h> +#include <vespa/searchlib/util/random_normal.h> + +namespace search { +namespace features { + +/** + * Implements the executor for the random normal feature outputting a + * random number drawn from the Gaussian distribution with the + * two arguments 'mean' and 'stddev'. + * The same hit always returns the same random number. + **/ +class RandomNormalStableExecutor : public fef::FeatureExecutor { +private: + RandomNormal _rnd; // seeded once per match + uint64_t _seed; + +public: + RandomNormalStableExecutor(uint64_t seed, double mean, double stddev); + void execute(uint32_t docId) override; +}; + + +/** + * Implements the blueprint for the random normal stable feature. + */ +class RandomNormalStableBlueprint : public fef::Blueprint { +private: + uint64_t _seed; + double _mean; + double _stddev; + +public: + RandomNormalStableBlueprint(); + + void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override; + fef::Blueprint::UP createInstance() const override; + fef::ParameterDescriptions getDescriptions() const override { + return fef::ParameterDescriptions(). + // Can run without parameters: + desc(). + + // Can run with two parameters (mean and stddev): + desc(). + number(). // mean + number(). // stddev + + // Can run with three parameters: + desc(). + number(). // mean + number(). // stddev + string(); // in order to name different features + } + + bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; + fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; +}; + + +} // namespace features +} // namespace search + diff --git a/searchlib/src/vespa/searchlib/features/setup.cpp b/searchlib/src/vespa/searchlib/features/setup.cpp index 867f058931f..1d3c59f5b3d 100644 --- a/searchlib/src/vespa/searchlib/features/setup.cpp +++ b/searchlib/src/vespa/searchlib/features/setup.cpp @@ -38,6 +38,7 @@ #include "querytermcountfeature.h" #include "randomfeature.h" #include "random_normal_feature.h" +#include "random_normal_stable_feature.h" #include "rankingexpressionfeature.h" #include "raw_score_feature.h" #include "reverseproximityfeature.h" @@ -100,6 +101,7 @@ void setup_search_features(fef::IBlueprintRegistry & registry) registry.addPrototype(Blueprint::SP(new QueryTermCountBlueprint())); registry.addPrototype(Blueprint::SP(new RandomBlueprint())); registry.addPrototype(Blueprint::SP(new RandomNormalBlueprint())); + registry.addPrototype(Blueprint::SP(new RandomNormalStableBlueprint())); registry.addPrototype(Blueprint::SP(new RawScoreBlueprint())); registry.addPrototype(Blueprint::SP(new SubqueriesBlueprint)); registry.addPrototype(Blueprint::SP(new TensorFromLabelsBlueprint())); diff --git a/searchlib/src/vespa/searchlib/fef/featurenamebuilder.cpp b/searchlib/src/vespa/searchlib/fef/featurenamebuilder.cpp index ec00b7d2f90..0eba912fafd 100644 --- a/searchlib/src/vespa/searchlib/fef/featurenamebuilder.cpp +++ b/searchlib/src/vespa/searchlib/fef/featurenamebuilder.cpp @@ -136,7 +136,7 @@ FeatureNameBuilder::buildName() const vespalib::string ret; if (!_baseName.empty()) { ret = _baseName; - if (!_parameters.empty() > 0) { + if (!_parameters.empty()) { ret += "("; for (uint32_t i = 0; i < _parameters.size(); ++i) { if (i > 0) { diff --git a/searchlib/src/vespa/searchlib/index/docbuilder.h b/searchlib/src/vespa/searchlib/index/docbuilder.h index 1d170483783..73c60304f50 100644 --- a/searchlib/src/vespa/searchlib/index/docbuilder.h +++ b/searchlib/src/vespa/searchlib/index/docbuilder.h @@ -4,6 +4,7 @@ #include "doctypebuilder.h" #include <vespa/document/datatype/datatypes.h> +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/annotation/annotation.h> #include <vespa/document/annotation/span.h> diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp index 226f41a53c0..3165cd9b68a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp @@ -4,10 +4,37 @@ #include <vespa/searchlib/fef/termfieldmatchdataposition.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.h> +#include <vespa/searchcommon/attribute/i_search_context.h> namespace search { namespace queryeval { +namespace { + +struct FakeContext : search::attribute::ISearchContext { + int32_t onFind(DocId, int32_t, int32_t &) const override { return -1; } + int32_t onFind(DocId, int32_t) const override { return -1; } + unsigned int approximateHits() const override { return 0; } + std::unique_ptr<SearchIterator> createIterator(fef::TermFieldMatchData *, bool) override { abort(); } + void fetchPostings(bool) override { } + bool valid() const override { return true; } + search::Int64Range getAsIntegerTerm() const override { abort(); } + const search::QueryTermBase &queryTerm() const override { abort(); } + const vespalib::string &attributeName() const override { abort(); } +}; + +} // namespace search::queryeval::<unnamed> + +void +FakeSearch::is_attr(bool value) +{ + if (value) { + _ctx = std::make_unique<FakeContext>(); + } else { + _ctx.reset(); + } +} + void FakeSearch::doSeek(uint32_t docid) { @@ -49,5 +76,5 @@ FakeSearch::visitMembers(vespalib::ObjectVisitor &visitor) const visit(visitor, "term", _term); } -} // namespace queryeval +} // namespace search::queryeval } // namespace search diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.h b/searchlib/src/vespa/searchlib/queryeval/fake_search.h index c320d497edc..aa6df480a21 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.h @@ -5,6 +5,7 @@ #include "searchiterator.h" #include "fake_result.h" #include <vespa/searchlib/fef/termfieldmatchdataarray.h> +#include <vespa/searchcommon/attribute/i_search_context.h> namespace search { namespace queryeval { @@ -18,6 +19,7 @@ private: FakeResult _result; uint32_t _offset; fef::TermFieldMatchDataArray _tfmda; + std::unique_ptr<attribute::ISearchContext> _ctx; bool valid() const { return _offset < _result.inspect().size(); } uint32_t currId() const { return _result.inspect()[_offset].docId; } @@ -34,10 +36,12 @@ public: { assert(_tfmda.size() == 1); } + void is_attr(bool value); void doSeek(uint32_t docid) override; void doUnpack(uint32_t docid) override; const PostingInfo *getPostingInfo() const override { return _result.postingInfo(); } void visitMembers(vespalib::ObjectVisitor &visitor) const override; + const attribute::ISearchContext *getAttributeSearchContext() const override { return _ctx.get(); } }; } // namespace queryeval diff --git a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp index bbfa487ae7d..a140fb146d5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp @@ -64,7 +64,9 @@ SimpleBlueprint::tag(const vespalib::string &t) SearchIterator::UP FakeBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool) const { - return std::make_unique<FakeSearch>(_tag, _field.getName(), _term, _result, tfmda); + auto result = std::make_unique<FakeSearch>(_tag, _field.getName(), _term, _result, tfmda); + result->is_attr(_is_attr); + return result; } FakeBlueprint::FakeBlueprint(const FieldSpec &field, const FakeResult &result) @@ -72,7 +74,8 @@ FakeBlueprint::FakeBlueprint(const FieldSpec &field, const FakeResult &result) _tag("<tag>"), _term("<term>"), _field(field), - _result(result) + _result(result), + _is_attr(false) { setEstimate(HitEstimate(result.inspect().size(), result.inspect().empty())); } diff --git a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h index 698e2235690..85d30aaf003 100644 --- a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h @@ -51,6 +51,7 @@ private: vespalib::string _term; FieldSpec _field; FakeResult _result; + bool _is_attr; protected: SearchIterator::UP @@ -66,6 +67,12 @@ public: } const vespalib::string &tag() const { return _tag; } + FakeBlueprint &is_attr(bool value) { + _is_attr = value; + return *this; + } + bool is_attr() const { return _is_attr; } + FakeBlueprint &term(const vespalib::string &t) { _term = t; return *this; diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_search.h b/searchlib/src/vespa/searchlib/queryeval/same_element_search.h index 6a116c76e73..1fd381eb1ae 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_search.h @@ -39,6 +39,7 @@ public: void doSeek(uint32_t docid) override; void doUnpack(uint32_t) override {} void visitMembers(vespalib::ObjectVisitor &visitor) const override; + const std::vector<SearchIterator::UP> &children() const { return _children; } }; } diff --git a/searchlib/src/vespa/searchlib/util/random_normal.h b/searchlib/src/vespa/searchlib/util/random_normal.h new file mode 100644 index 00000000000..74596066312 --- /dev/null +++ b/searchlib/src/vespa/searchlib/util/random_normal.h @@ -0,0 +1,67 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/util/rand48.h> +#include <cmath> + +namespace search { + +/** + * Draws a random number from the Gaussian distribution + * using the Marsaglia polar method. + */ +class RandomNormal +{ +private: + Rand48 _rnd; + double _mean; + double _stddev; + + bool _useSpare; + bool _hasSpare; + feature_t _spare; + + feature_t nextUniform() { + return (_rnd.lrand48() / (feature_t)0x80000000u) * 2.0 - 1.0; + } + +public: + RandomNormal(double mean, double stddev, bool useSpare = true) : + _rnd(), + _mean(mean), + _stddev(stddev), + _useSpare(useSpare), + _hasSpare(false), + _spare(0.0) + {} + + void seed(long seed) { + _rnd.srand48(seed); + } + + feature_t next() { + feature_t result = _spare; + if (_useSpare && _hasSpare) { + _hasSpare = false; + } else { + _hasSpare = true; + + feature_t u, v, s; + do { + u = nextUniform(); + v = nextUniform(); + s = u * u + v * v; + } while ( (s >= 1.0) || (s == 0.0) ); + s = std::sqrt(-2.0 * std::log(s) / s); + + _spare = v * s; // saved for next invocation + result = u * s; + } + return _mean + _stddev * result; + } + +}; + +} // search + diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java index bd2658db8aa..7c78d61da30 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; /** * Responsible for monitoring a whole application using /state/v1/health. @@ -37,7 +38,13 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos private final Map<ServiceId, HealthMonitor> healthMonitors; public static ApplicationHealthMonitor startMonitoring(ApplicationInfo application) { - return new ApplicationHealthMonitor(makeHealthMonitors(application)); + return startMonitoring(application, HealthMonitor::new); + } + + /** For testing. */ + static ApplicationHealthMonitor startMonitoring(ApplicationInfo application, + Function<HealthEndpoint, HealthMonitor> mapper) { + return new ApplicationHealthMonitor(makeHealthMonitors(application, mapper)); } private ApplicationHealthMonitor(Map<ServiceId, HealthMonitor> healthMonitors) { @@ -64,7 +71,8 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos healthMonitors.clear(); } - private static Map<ServiceId, HealthMonitor> makeHealthMonitors(ApplicationInfo application) { + private static Map<ServiceId, HealthMonitor> makeHealthMonitors( + ApplicationInfo application, Function<HealthEndpoint, HealthMonitor> monitorFactory) { Map<ServiceId, HealthMonitor> healthMonitors = new HashMap<>(); for (HostInfo hostInfo : application.getModel().getHosts()) { for (ServiceInfo serviceInfo : hostInfo.getServices()) { @@ -73,7 +81,8 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos application, hostInfo, serviceInfo, - portInfo) + portInfo, + monitorFactory) .ifPresent(healthMonitor -> healthMonitors.put( ApplicationInstanceGenerator.getServiceId(application, serviceInfo), healthMonitor)); @@ -87,14 +96,14 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos ApplicationInfo applicationInfo, HostInfo hostInfo, ServiceInfo serviceInfo, - PortInfo portInfo) { + PortInfo portInfo, + Function<HealthEndpoint, HealthMonitor> monitorFactory) { if (portInfo.getTags().containsAll(PORT_TAGS_HEALTH)) { HostName hostname = HostName.from(hostInfo.getHostname()); HealthEndpoint endpoint = HealthEndpoint.forHttp(hostname, portInfo.getPort()); - // todo: make HealthMonitor - // HealthMonitor healthMonitor = new HealthMonitor(endpoint); - // healthMonitor.startMonitoring(); - return Optional.empty(); + HealthMonitor healthMonitor = monitorFactory.apply(endpoint); + healthMonitor.startMonitoring(); + return Optional.of(healthMonitor); } return Optional.empty(); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java index 43a02a385be..c8b2d5d25f7 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java @@ -14,7 +14,6 @@ import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.ConnectionKeepAliveStrategy; import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; import org.apache.http.impl.client.HttpClients; @@ -23,8 +22,16 @@ import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; import javax.net.ssl.SSLContext; +import java.util.function.Function; +import java.util.function.Supplier; + +import static com.yahoo.yolean.Exceptions.uncheck; /** + * Health client + * + * NOT thread-safe. + * * @author hakon */ public class HealthClient implements AutoCloseable, ServiceIdentityProvider.Listener { @@ -47,27 +54,76 @@ public class HealthClient implements AutoCloseable, ServiceIdentityProvider.List }; private final HealthEndpoint endpoint; + private final Supplier<CloseableHttpClient> clientSupplier; + private final Function<HttpEntity, String> getContentFunction; - private volatile CloseableHttpClient httpClient; + private CloseableHttpClient httpClient = null; public HealthClient(HealthEndpoint endpoint) { + this(endpoint, + () -> makeCloseableHttpClient(endpoint), + entity -> uncheck(() -> EntityUtils.toString(entity))); + } + + /** For testing. */ + HealthClient(HealthEndpoint endpoint, + Supplier<CloseableHttpClient> clientSupplier, + Function<HttpEntity, String> getContentFunction) { this.endpoint = endpoint; + this.clientSupplier = clientSupplier; + this.getContentFunction = getContentFunction; } public void start() { - endpoint.getServiceIdentityProvider().ifPresent(provider -> { - onCredentialsUpdate(provider.getIdentitySslContext(), null); - provider.addIdentityListener(this); - }); + updateHttpClient(); + endpoint.registerListener(this); } @Override public void onCredentialsUpdate(SSLContext sslContext, AthenzService ignored) { - SSLConnectionSocketFactory socketFactory = - new SSLConnectionSocketFactory(sslContext, endpoint.getHostnameVerifier().orElse(null)); + updateHttpClient(); + } + + public HealthEndpoint getEndpoint() { + return endpoint; + } + public HealthInfo getHealthInfo() { + try { + return probeHealth(); + } catch (Exception e) { + return HealthInfo.fromException(e); + } + } + + @Override + public void close() { + endpoint.removeListener(this); + + if (httpClient != null) { + try { + httpClient.close(); + } catch (Exception e) { + // ignore + } + httpClient = null; + } + } + + private void updateHttpClient() { + CloseableHttpClient httpClient = clientSupplier.get(); + + if (this.httpClient != null) { + // Note: close() can be called any number of times. + uncheck(() -> this.httpClient.close()); + } + + this.httpClient = httpClient; + } + + private static CloseableHttpClient makeCloseableHttpClient(HealthEndpoint endpoint) { Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() - .register("https", socketFactory) + .register(endpoint.getStateV1HealthUrl().getProtocol(), endpoint.getConnectionSocketFactory()) .build(); HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry); @@ -78,7 +134,7 @@ public class HealthClient implements AutoCloseable, ServiceIdentityProvider.List .setSocketTimeout(DEFAULT_TIMEOUT_MILLIS) // waiting for data .build(); - this.httpClient = HttpClients.custom() + return HttpClients.custom() .setKeepAliveStrategy(KEEP_ALIVE_STRATEGY) .setConnectionManager(connectionManager) .disableAutomaticRetries() @@ -86,54 +142,33 @@ public class HealthClient implements AutoCloseable, ServiceIdentityProvider.List .build(); } - public HealthInfo getHealthInfo() { - try { - return probeHealth(); - } catch (Exception e) { - return HealthInfo.fromException(e); - } - } - - @Override - public void close() { - endpoint.getServiceIdentityProvider().ifPresent(provider -> provider.removeIdentityListener(this)); - - try { - httpClient.close(); - } catch (Exception e) { - // ignore - } - httpClient = null; - } - private HealthInfo probeHealth() throws Exception { HttpGet httpget = new HttpGet(endpoint.getStateV1HealthUrl().toString()); - CloseableHttpResponse httpResponse; CloseableHttpClient httpClient = this.httpClient; if (httpClient == null) { - throw new IllegalStateException("HTTP client has closed"); - } - - httpResponse = httpClient.execute(httpget); - - int httpStatusCode = httpResponse.getStatusLine().getStatusCode(); - if (httpStatusCode < 200 || httpStatusCode >= 300) { - return HealthInfo.fromBadHttpStatusCode(httpStatusCode); - } - - HttpEntity bodyEntity = httpResponse.getEntity(); - long contentLength = bodyEntity.getContentLength(); - if (contentLength > MAX_CONTENT_LENGTH) { - throw new IllegalArgumentException("Content too long: " + contentLength + " bytes"); + throw new IllegalStateException("HTTP client never started or has closed"); } - String body = EntityUtils.toString(bodyEntity); - HealthResponse healthResponse = mapper.readValue(body, HealthResponse.class); - if (healthResponse.status == null || healthResponse.status.code == null) { - return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS); - } else { - return HealthInfo.fromHealthStatusCode(healthResponse.status.code); + try (CloseableHttpResponse httpResponse = httpClient.execute(httpget)) { + int httpStatusCode = httpResponse.getStatusLine().getStatusCode(); + if (httpStatusCode < 200 || httpStatusCode >= 300) { + return HealthInfo.fromBadHttpStatusCode(httpStatusCode); + } + + HttpEntity bodyEntity = httpResponse.getEntity(); + long contentLength = bodyEntity.getContentLength(); + if (contentLength > MAX_CONTENT_LENGTH) { + throw new IllegalArgumentException("Content too long: " + contentLength + " bytes"); + } + String body = getContentFunction.apply(bodyEntity); + HealthResponse healthResponse = mapper.readValue(body, HealthResponse.class); + + if (healthResponse.status == null || healthResponse.status.code == null) { + return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS); + } else { + return HealthInfo.fromHealthStatusCode(healthResponse.status.code); + } } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java index e9d17a9ab70..38139d28d7f 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java @@ -5,25 +5,22 @@ import com.yahoo.config.provision.HostName; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; +import org.apache.http.conn.socket.ConnectionSocketFactory; import javax.net.ssl.HostnameVerifier; import java.net.URL; import java.util.Collections; -import java.util.Optional; import static com.yahoo.yolean.Exceptions.uncheck; /** * @author hakon */ -class HealthEndpoint { - private final URL url; - private final Optional<HostnameVerifier> hostnameVerifier; - private final Optional<ServiceIdentityProvider> serviceIdentityProvider; +public interface HealthEndpoint { static HealthEndpoint forHttp(HostName hostname, int port) { URL url = uncheck(() -> new URL("http", hostname.value(), port, "/state/v1/health")); - return new HealthEndpoint(url, Optional.empty(), Optional.empty()); + return new HttpHealthEndpoint(url); } static HealthEndpoint forHttps(HostName hostname, @@ -32,26 +29,12 @@ class HealthEndpoint { AthenzIdentity remoteIdentity) { URL url = uncheck(() -> new URL("https", hostname.value(), port, "/state/v1/health")); HostnameVerifier peerVerifier = new AthenzIdentityVerifier(Collections.singleton(remoteIdentity)); - return new HealthEndpoint(url, Optional.of(serviceIdentityProvider), Optional.of(peerVerifier)); + return new HttpsHealthEndpoint(url, serviceIdentityProvider, peerVerifier); } - private HealthEndpoint(URL url, - Optional<ServiceIdentityProvider> serviceIdentityProvider, - Optional<HostnameVerifier> hostnameVerifier) { - this.url = url; - this.serviceIdentityProvider = serviceIdentityProvider; - this.hostnameVerifier = hostnameVerifier; - } - - public URL getStateV1HealthUrl() { - return url; - } - - public Optional<ServiceIdentityProvider> getServiceIdentityProvider() { - return serviceIdentityProvider; - } - - public Optional<HostnameVerifier> getHostnameVerifier() { - return hostnameVerifier; - } + URL getStateV1HealthUrl(); + ConnectionSocketFactory getConnectionSocketFactory(); + void registerListener(ServiceIdentityProvider.Listener listener); + void removeListener(ServiceIdentityProvider.Listener listener); + String description(); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java index a3fe3cb3106..8b724afba5f 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java @@ -50,7 +50,17 @@ public class HealthInfo { return healthStatusCode.map(UP_STATUS_CODE::equals).orElse(false); } - public ServiceStatus toSerivceStatus() { + public ServiceStatus toServiceStatus() { + // Bootstrapping ServiceStatus: To avoid thundering herd problem at startup, + // the clients will not fetch the health immediately. What should the ServiceStatus + // be before the first health has been fetched? + // + // NOT_CHECKED: Logically the right thing, but if an Orchestrator gets a suspend request + // in this window, and another service within the cluster is down, it ends up allowing + // suspension when it shouldn't have done so. + // + // DOWN: Only safe initial value, possibly except if the first initial delay is long, + // as that could indicate it has been down for too long. return isHealthy() ? ServiceStatus.UP : ServiceStatus.DOWN; } @@ -65,7 +75,7 @@ public class HealthInfo { } else if (healthStatusCode.isPresent()) { return "Bad health status code '" + healthStatusCode.get() + "'"; } else if (exception.isPresent()) { - return Exceptions.toMessageString(exception.get()); + return "Exception: " + Exceptions.toMessageString(exception.get()); } else if (httpStatusCode.isPresent()) { return "Bad HTTP response status code " + httpStatusCode.getAsInt(); } else { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java index fd809b32918..2574f782afb 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java @@ -13,40 +13,48 @@ import java.util.logging.Logger; /** * Used to monitor the health of a single URL endpoint. * + * <p>Must be closed on successful start of monitoring ({} + * + * <p>Thread-safe + * * @author hakon */ public class HealthMonitor implements AutoCloseable { private static final Logger logger = Logger.getLogger(HealthMonitor.class.getName()); - private static final Duration DELAY = Duration.ofSeconds(20); + + /** The duration between each health request. */ + private static final Duration DEFAULT_DELAY = Duration.ofSeconds(10); + // About 'static': Javadoc says "Instances of java.util.Random are threadsafe." private static final Random random = new Random(); private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); private final HealthClient healthClient; + private final Duration delay; private volatile HealthInfo lastHealthInfo = HealthInfo.empty(); public HealthMonitor(HealthEndpoint stateV1HealthEndpoint) { - this.healthClient = new HealthClient(stateV1HealthEndpoint); + this(new HealthClient(stateV1HealthEndpoint), DEFAULT_DELAY); } /** For testing. */ - HealthMonitor(HealthClient healthClient) { + HealthMonitor(HealthClient healthClient, Duration delay) { this.healthClient = healthClient; + this.delay = delay; } public void startMonitoring() { healthClient.start(); executor.scheduleWithFixedDelay( this::updateSynchronously, - initialDelayInSeconds(DELAY.getSeconds()), - DELAY.getSeconds(), - TimeUnit.SECONDS); + initialDelayInMillis(delay.toMillis()), + delay.toMillis(), + TimeUnit.MILLISECONDS); } public ServiceStatus getStatus() { - // todo: return lastHealthInfo.toServiceStatus(); - return ServiceStatus.NOT_CHECKED; + return lastHealthInfo.toServiceStatus(); } @Override @@ -63,11 +71,21 @@ public class HealthMonitor implements AutoCloseable { healthClient.close(); } - private long initialDelayInSeconds(long maxInitialDelayInSeconds) { - return random.nextLong() % maxInitialDelayInSeconds; + private long initialDelayInMillis(long maxInitialDelayInMillis) { + if (maxInitialDelayInMillis >= Integer.MAX_VALUE) { + throw new IllegalArgumentException("Max initial delay is out of bounds: " + maxInitialDelayInMillis); + } + + return (long) random.nextInt((int) maxInitialDelayInMillis); } private void updateSynchronously() { - lastHealthInfo = healthClient.getHealthInfo(); + try { + lastHealthInfo = healthClient.getHealthInfo(); + } catch (Throwable t) { + // An uncaught exception will kill the executor.scheduleWithFixedDelay thread! + logger.log(LogLevel.WARNING, "Failed to get health info for " + + healthClient.getEndpoint().description(), t); + } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java index 473ef5e3a94..383cb6961a7 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java @@ -9,17 +9,20 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; import com.yahoo.vespa.service.monitor.application.ZoneApplication; import com.yahoo.vespa.service.monitor.internal.MonitorManager; -import java.util.HashMap; -import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** + * Manages all /state/v1/health related monitoring. + * * @author hakon */ public class HealthMonitorManager implements MonitorManager { - private final Map<ApplicationId, ApplicationHealthMonitor> healthMonitors = new HashMap<>(); + private final ConcurrentHashMap<ApplicationId, ApplicationHealthMonitor> healthMonitors = + new ConcurrentHashMap<>(); private final ConfigserverConfig configserverConfig; @Inject @@ -29,7 +32,7 @@ public class HealthMonitorManager implements MonitorManager { @Override public void applicationActivated(ApplicationInfo application) { - if (applicationMonitored(application.getApplicationId())) { + if (applicationMonitoredForHealth(application.getApplicationId())) { ApplicationHealthMonitor monitor = ApplicationHealthMonitor.startMonitoring(application); healthMonitors.put(application.getApplicationId(), monitor); @@ -38,11 +41,9 @@ public class HealthMonitorManager implements MonitorManager { @Override public void applicationRemoved(ApplicationId id) { - if (applicationMonitored(id)) { - ApplicationHealthMonitor monitor = healthMonitors.remove(id); - if (monitor != null) { - monitor.close(); - } + ApplicationHealthMonitor monitor = healthMonitors.remove(id); + if (monitor != null) { + monitor.close(); } } @@ -58,11 +59,15 @@ public class HealthMonitorManager implements MonitorManager { return ServiceStatus.UP; } - return ServiceStatus.NOT_CHECKED; + ApplicationHealthMonitor monitor = healthMonitors.get(applicationId); + if (monitor == null) { + return ServiceStatus.NOT_CHECKED; + } + + return monitor.getStatus(applicationId, clusterId, serviceType, configId); } - private boolean applicationMonitored(ApplicationId id) { - // todo: health-check config server - return false; + private boolean applicationMonitoredForHealth(ApplicationId id) { + return id.equals(ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId()); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpHealthEndpoint.java new file mode 100644 index 00000000000..254cb9785e1 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpHealthEndpoint.java @@ -0,0 +1,44 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.health; + +import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; + +import java.net.URL; + +/** + * @author hakon + */ +class HttpHealthEndpoint implements HealthEndpoint { + private final URL url; + private final ConnectionSocketFactory socketFactory; + + HttpHealthEndpoint(URL url) { + this.url = url; + this.socketFactory = PlainConnectionSocketFactory.getSocketFactory(); + } + + @Override + public URL getStateV1HealthUrl() { + return url; + } + + @Override + public ConnectionSocketFactory getConnectionSocketFactory() { + return socketFactory; + } + + @Override + public void registerListener(ServiceIdentityProvider.Listener listener) { + } + + @Override + public void removeListener(ServiceIdentityProvider.Listener listener) { + } + + @Override + public String description() { + return url.toString(); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpsHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpsHealthEndpoint.java new file mode 100644 index 00000000000..f1ebb80f500 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpsHealthEndpoint.java @@ -0,0 +1,53 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.health; + +import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import java.net.URL; + +/** + * @author hakon + */ +public class HttpsHealthEndpoint implements HealthEndpoint { + private final URL url; + private final HostnameVerifier hostnameVerifier; + private final ServiceIdentityProvider serviceIdentityProvider; + + HttpsHealthEndpoint(URL url, + ServiceIdentityProvider serviceIdentityProvider, + HostnameVerifier hostnameVerifier) { + this.url = url; + this.serviceIdentityProvider = serviceIdentityProvider; + this.hostnameVerifier = hostnameVerifier; + } + + @Override + public URL getStateV1HealthUrl() { + return url; + } + + @Override + public ConnectionSocketFactory getConnectionSocketFactory() { + SSLContext sslContext = serviceIdentityProvider.getIdentitySslContext(); + return new SSLConnectionSocketFactory(sslContext, hostnameVerifier); + } + + @Override + public void registerListener(ServiceIdentityProvider.Listener listener) { + serviceIdentityProvider.addIdentityListener(listener); + } + + @Override + public void removeListener(ServiceIdentityProvider.Listener listener) { + serviceIdentityProvider.removeIdentityListener(listener); + } + + @Override + public String description() { + return url.toString(); + } +} diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java index 51b0503565f..b0fdb14726f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java @@ -6,19 +6,121 @@ import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; import com.yahoo.vespa.service.monitor.internal.ConfigserverUtil; import org.junit.Test; -import static com.yahoo.vespa.applicationmodel.ServiceStatus.NOT_CHECKED; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; + import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class ApplicationHealthMonitorTest { @Test public void sanityCheck() { - ApplicationHealthMonitor monitor = ApplicationHealthMonitor.startMonitoring( - ConfigserverUtil.makeExampleConfigServer()); - ServiceStatus status = monitor.getStatus( + MonitorFactory monitorFactory = new MonitorFactory(); + + HealthMonitor monitor1 = mock(HealthMonitor.class); + HealthMonitor monitor2 = mock(HealthMonitor.class); + HealthMonitor monitor3 = mock(HealthMonitor.class); + + monitorFactory.expectEndpoint("http://cfg1:19071/state/v1/health", monitor1); + monitorFactory.expectEndpoint("http://cfg2:19071/state/v1/health", monitor2); + monitorFactory.expectEndpoint("http://cfg3:19071/state/v1/health", monitor3); + + when(monitor1.getStatus()).thenReturn(ServiceStatus.UP); + when(monitor2.getStatus()).thenReturn(ServiceStatus.DOWN); + when(monitor3.getStatus()).thenReturn(ServiceStatus.NOT_CHECKED); + + ApplicationHealthMonitor applicationMonitor = ApplicationHealthMonitor.startMonitoring( + ConfigserverUtil.makeExampleConfigServer(), + monitorFactory); + + ServiceStatus status1 = applicationMonitor.getStatus( ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), ConfigServerApplication.CLUSTER_ID, ConfigServerApplication.SERVICE_TYPE, ConfigServerApplication.configIdFrom(0)); - assertEquals(NOT_CHECKED, status); + assertEquals(ServiceStatus.UP, status1); + + ServiceStatus status2 = applicationMonitor.getStatus( + ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), + ConfigServerApplication.CLUSTER_ID, + ConfigServerApplication.SERVICE_TYPE, + ConfigServerApplication.configIdFrom(1)); + assertEquals(ServiceStatus.DOWN, status2); + + ServiceStatus status3 = applicationMonitor.getStatus( + ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), + ConfigServerApplication.CLUSTER_ID, + ConfigServerApplication.SERVICE_TYPE, + ConfigServerApplication.configIdFrom(2)); + assertEquals(ServiceStatus.NOT_CHECKED, status3); + } + + private static class MonitorFactory implements Function<HealthEndpoint, HealthMonitor> { + private Map<String, EndpointInfo> endpointMonitors = new HashMap<>(); + + public void expectEndpoint(String url, HealthMonitor monitorToReturn) { + endpointMonitors.put(url, new EndpointInfo(url, monitorToReturn)); + } + + @Override + public HealthMonitor apply(HealthEndpoint endpoint) { + String url = endpoint.getStateV1HealthUrl().toString(); + EndpointInfo info = endpointMonitors.get(url); + if (info == null) { + throw new IllegalArgumentException("Endpoint not expected: " + url); + } + + if (info.isEndpointDiscovered()) { + throw new IllegalArgumentException("A HealthMonitor has already been created to " + url); + } + + info.setEndpointDiscovered(true); + + return info.getMonitorToReturn(); + } + } + + private static class EndpointInfo { + private final String url; + private final HealthMonitor monitorToReturn; + + private boolean endpointDiscovered = false; + + private EndpointInfo(String url, HealthMonitor monitorToReturn) { + this.url = url; + this.monitorToReturn = monitorToReturn; + } + + public String getUrl() { + return url; + } + + public boolean isEndpointDiscovered() { + return endpointDiscovered; + } + + public void setEndpointDiscovered(boolean endpointDiscovered) { + this.endpointDiscovered = endpointDiscovered; + } + + public HealthMonitor getMonitorToReturn() { + return monitorToReturn; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EndpointInfo that = (EndpointInfo) o; + return Objects.equals(url, that.url); + } + + @Override + public int hashCode() { + return Objects.hash(url); + } } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthClientTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthClientTest.java new file mode 100644 index 00000000000..c3e06faaf92 --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthClientTest.java @@ -0,0 +1,165 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.health; + +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.applicationmodel.ServiceStatus; +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.impl.client.CloseableHttpClient; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class HealthClientTest { + @Test + public void successfulRequestResponse() throws IOException { + HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + " \"metrics\": {\n" + + " \"snapshot\": {\n" + + " \"from\": 1.528789829249E9,\n" + + " \"to\": 1.528789889249E9\n" + + " }\n" + + " },\n" + + " \"status\": {\"code\": \"up\"},\n" + + " \"time\": 1528789889364\n" + + "}"); + assertTrue(info.isHealthy()); + assertEquals(ServiceStatus.UP, info.toServiceStatus()); + } + + @Test + public void notUpResponse() throws IOException { + HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + " \"metrics\": {\n" + + " \"snapshot\": {\n" + + " \"from\": 1.528789829249E9,\n" + + " \"to\": 1.528789889249E9\n" + + " }\n" + + " },\n" + + " \"status\": {\"code\": \"initializing\"},\n" + + " \"time\": 1528789889364\n" + + "}"); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertEquals("Bad health status code 'initializing'", info.toString()); + } + + @Test + public void noCodeInResponse() throws IOException { + HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + " \"metrics\": {\n" + + " \"snapshot\": {\n" + + " \"from\": 1.528789829249E9,\n" + + " \"to\": 1.528789889249E9\n" + + " }\n" + + " },\n" + + " \"status\": {\"foo\": \"bar\"},\n" + + " \"time\": 1528789889364\n" + + "}"); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertEquals("Bad health status code 'down'", info.toString()); + } + + @Test + public void noStatusInResponse() throws IOException { + HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + " \"metrics\": {\n" + + " \"snapshot\": {\n" + + " \"from\": 1.528789829249E9,\n" + + " \"to\": 1.528789889249E9\n" + + " }\n" + + " },\n" + + " \"time\": 1528789889364\n" + + "}"); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertEquals("Bad health status code 'down'", info.toString()); + } + + @Test + public void badJson() throws IOException { + HealthInfo info = getHealthInfoFromJsonResponse("} foo bar"); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertTrue(info.toString().startsWith("Exception: Unexpected close marker '}': ")); + } + + private HealthInfo getHealthInfoFromJsonResponse(String content) + throws IOException { + HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071); + CloseableHttpClient client = mock(CloseableHttpClient.class); + + CloseableHttpResponse response = mock(CloseableHttpResponse.class); + when(client.execute(any())).thenReturn(response); + + StatusLine statusLine = mock(StatusLine.class); + when(response.getStatusLine()).thenReturn(statusLine); + + when(statusLine.getStatusCode()).thenReturn(200); + + HttpEntity httpEntity = mock(HttpEntity.class); + when(response.getEntity()).thenReturn(httpEntity); + + try (HealthClient healthClient = new HealthClient(endpoint, () -> client, entry -> content)) { + healthClient.start(); + + when(httpEntity.getContentLength()).thenReturn((long) content.length()); + return healthClient.getHealthInfo(); + } + } + + @Test + public void testRequestException() throws IOException { + HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071); + CloseableHttpClient client = mock(CloseableHttpClient.class); + + when(client.execute(any())).thenThrow(new ConnectTimeoutException("exception string")); + + try (HealthClient healthClient = new HealthClient(endpoint, () -> client, entry -> "")) { + healthClient.start(); + HealthInfo info = healthClient.getHealthInfo(); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertEquals("Exception: exception string", info.toString()); + } + } + + @Test + public void testBadHttpResponseCode() + throws IOException { + HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071); + CloseableHttpClient client = mock(CloseableHttpClient.class); + + CloseableHttpResponse response = mock(CloseableHttpResponse.class); + when(client.execute(any())).thenReturn(response); + + StatusLine statusLine = mock(StatusLine.class); + when(response.getStatusLine()).thenReturn(statusLine); + + when(statusLine.getStatusCode()).thenReturn(500); + + HttpEntity httpEntity = mock(HttpEntity.class); + when(response.getEntity()).thenReturn(httpEntity); + + String content = "{}"; + try (HealthClient healthClient = new HealthClient(endpoint, () -> client, entry -> content)) { + healthClient.start(); + + when(httpEntity.getContentLength()).thenReturn((long) content.length()); + HealthInfo info = healthClient.getHealthInfo(); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertEquals("Bad HTTP response status code 500", info.toString()); + } + } +}
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java index cca1530ad97..2a203027353 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java @@ -4,18 +4,36 @@ package com.yahoo.vespa.service.monitor.internal.health; import com.yahoo.vespa.applicationmodel.ServiceStatus; import org.junit.Test; -import java.net.MalformedURLException; +import java.time.Duration; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class HealthMonitorTest { @Test - public void basicTests() throws MalformedURLException { + public void initiallyDown() { HealthClient healthClient = mock(HealthClient.class); - try (HealthMonitor monitor = new HealthMonitor(healthClient)) { + try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofHours(12))) { monitor.startMonitoring(); - assertEquals(ServiceStatus.NOT_CHECKED, monitor.getStatus()); + assertEquals(ServiceStatus.DOWN, monitor.getStatus()); + } + } + + @Test + public void eventuallyUp() { + HealthClient healthClient = mock(HealthClient.class); + when(healthClient.getHealthInfo()).thenReturn(HealthInfo.fromHealthStatusCode(HealthInfo.UP_STATUS_CODE)); + try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofMillis(10))) { + monitor.startMonitoring(); + + while (monitor.getStatus() != ServiceStatus.UP) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + // ignore + } + } } } }
\ No newline at end of file diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp index 52c9def3036..7e14fbc0014 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp +++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp @@ -48,7 +48,7 @@ Register::Register() : _listByName() { } -Register::~Register() { } +Register::~Register() = default; bool Register::erase(Identifiable::RuntimeClass * c) { diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index d12e6b90b2a..829a080ad01 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -723,6 +723,7 @@ public: auto createUpdateCommand(const document::BucketId& bucket) const { auto update = std::make_shared<document::DocumentUpdate>( + _self._node->getTestDocMan().getTypeRepo(), *_self._node->getTestDocMan().getTypeRepo() .getDocumentType("testdoctype1"), document::DocumentId("id:foo:testdoctype1::bar2")); diff --git a/storage/src/tests/distributor/externaloperationhandlertest.cpp b/storage/src/tests/distributor/externaloperationhandlertest.cpp index 81b0293b0c0..54aca78d13d 100644 --- a/storage/src/tests/distributor/externaloperationhandlertest.cpp +++ b/storage/src/tests/distributor/externaloperationhandlertest.cpp @@ -200,6 +200,7 @@ std::shared_ptr<api::UpdateCommand> ExternalOperationHandlerTest::makeUpdateComm const vespalib::string& doc_type, const vespalib::string& id) const { auto update = std::make_shared<document::DocumentUpdate>( + _testDocMan.getTypeRepo(), *_testDocMan.getTypeRepo().getDocumentType(doc_type), document::DocumentId(id)); return std::make_shared<api::UpdateCommand>( diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp index 28602124045..ea2cc00c642 100644 --- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -294,7 +294,7 @@ TwoPhaseUpdateOperationTest::sendUpdate(const std::string& bucketState, document::DocumentUpdate::SP update; if (!options._withError) { update = std::make_shared<document::DocumentUpdate>( - *_doc_type, + *_repo, *_doc_type, document::DocumentId(document::DocIdString("test", "test"))); document::FieldUpdate fup(_doc_type->getField("headerval")); fup.addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10)); @@ -304,7 +304,7 @@ TwoPhaseUpdateOperationTest::sendUpdate(const std::string& bucketState, // part of the Get. Just a sneaky way to force an eval error. auto* badDocType = _repo->getDocumentType("testdoctype2"); update = std::make_shared<document::DocumentUpdate>( - *badDocType, + *_repo, *badDocType, document::DocumentId(document::DocIdString("test", "test"))); document::FieldUpdate fup(badDocType->getField("onlyinchild")); fup.addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10)); diff --git a/storage/src/tests/distributor/updateoperationtest.cpp b/storage/src/tests/distributor/updateoperationtest.cpp index 92474728957..9ce862f5db8 100644 --- a/storage/src/tests/distributor/updateoperationtest.cpp +++ b/storage/src/tests/distributor/updateoperationtest.cpp @@ -75,8 +75,7 @@ std::shared_ptr<UpdateOperation> UpdateOperation_Test::sendUpdate(const std::string& bucketState) { document::DocumentUpdate::SP update( - new document::DocumentUpdate( - *_html_type, + new document::DocumentUpdate(*_repo, *_html_type, document::DocumentId(document::DocIdString("test", "test")))); _bId = getExternalOperationHandler().getBucketId(update->getId()); diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 838df87662f..369e820f987 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -2725,6 +2725,7 @@ void FileStorManagerTest::update_command_size_is_added_to_metric() { document::BucketId bucket(16, 4000); createBucket(bucket, 0); auto update = std::make_shared<document::DocumentUpdate>( + _node->getTestDocMan().getTypeRepo(), _node->getTestDocMan().createRandomDocument()->getType(), document::DocumentId("id:foo:testdoctype1::bar")); auto cmd = std::make_shared<api::UpdateCommand>( diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index f15921e447d..368eae104e8 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -223,16 +223,11 @@ PersistenceTestUtils::doGetOnDisk( } document::DocumentUpdate::SP -PersistenceTestUtils::createBodyUpdate( - const document::DocumentId& docId, - const document::FieldValue& updateValue) +PersistenceTestUtils::createBodyUpdate(const document::DocumentId& docId, const document::FieldValue& updateValue) { - const DocumentType* docType(_env->_component.getTypeRepo() - ->getDocumentType("testdoctype1")); - document::DocumentUpdate::SP update( - new document::DocumentUpdate(*docType, docId)); - std::shared_ptr<document::AssignValueUpdate> assignUpdate( - new document::AssignValueUpdate(updateValue)); + const DocumentType* docType(_env->_component.getTypeRepo()->getDocumentType("testdoctype1")); + document::DocumentUpdate::SP update(new document::DocumentUpdate(*_env->_component.getTypeRepo(), *docType, docId)); + std::shared_ptr<document::AssignValueUpdate> assignUpdate(new document::AssignValueUpdate(updateValue)); document::FieldUpdate fieldUpdate(docType->getField("content")); fieldUpdate.addUpdate(*assignUpdate); update->addUpdate(fieldUpdate); @@ -240,16 +235,11 @@ PersistenceTestUtils::createBodyUpdate( } document::DocumentUpdate::SP -PersistenceTestUtils::createHeaderUpdate( - const document::DocumentId& docId, - const document::FieldValue& updateValue) +PersistenceTestUtils::createHeaderUpdate(const document::DocumentId& docId, const document::FieldValue& updateValue) { - const DocumentType* docType(_env->_component.getTypeRepo() - ->getDocumentType("testdoctype1")); - document::DocumentUpdate::SP update( - new document::DocumentUpdate(*docType, docId)); - std::shared_ptr<document::AssignValueUpdate> assignUpdate( - new document::AssignValueUpdate(updateValue)); + const DocumentType* docType(_env->_component.getTypeRepo()->getDocumentType("testdoctype1")); + document::DocumentUpdate::SP update(new document::DocumentUpdate(*_env->_component.getTypeRepo(), *docType, docId)); + std::shared_ptr<document::AssignValueUpdate> assignUpdate(new document::AssignValueUpdate(updateValue)); document::FieldUpdate fieldUpdate(docType->getField("headerval")); fieldUpdate.addUpdate(*assignUpdate); update->addUpdate(fieldUpdate); diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index c729df1e7eb..686e10ba5ef 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -189,7 +189,7 @@ std::unique_ptr<api::UpdateCommand> TestAndSetTest::conditional_update_test( { putTestDocument(matchingHeader, timestampOne); - auto docUpdate = std::make_shared<document::DocumentUpdate>(testDoc->getType(), testDocId); + auto docUpdate = std::make_shared<document::DocumentUpdate>(_env->_testDocMan.getTypeRepo(), testDoc->getType(), testDocId); auto fieldUpdate = document::FieldUpdate(testDoc->getField("content")); fieldUpdate.addUpdate(document::AssignValueUpdate(NEW_CONTENT)); docUpdate->addUpdate(fieldUpdate); diff --git a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp index 7df598bed97..98ad8761736 100644 --- a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp +++ b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp @@ -597,25 +597,19 @@ ChangedBucketOwnershipHandlerTest::testAbortOutdatedPutOperation() void ChangedBucketOwnershipHandlerTest::testAbortOutdatedUpdateCommand() { - const document::DocumentType* docType(_testDocRepo.getTypeRepo() - .getDocumentType("testdoctype1")); + const document::DocumentType* docType(_testDocRepo.getTypeRepo().getDocumentType("testdoctype1")); document::DocumentId docId("id:foo:testdoctype1::bar"); - document::DocumentUpdate::SP update( - std::make_shared<document::DocumentUpdate>(*docType, docId)); - CPPUNIT_ASSERT(changeAbortsMessage<api::UpdateCommand>( - getBucketToAbort(), update, api::Timestamp(1234))); - CPPUNIT_ASSERT(!changeAbortsMessage<api::UpdateCommand>( - getBucketToAllow(), update, api::Timestamp(1234))); + auto update(std::make_shared<document::DocumentUpdate>(_testDocRepo.getTypeRepo(), *docType, docId)); + CPPUNIT_ASSERT(changeAbortsMessage<api::UpdateCommand>(getBucketToAbort(), update, api::Timestamp(1234))); + CPPUNIT_ASSERT(!changeAbortsMessage<api::UpdateCommand>(getBucketToAllow(), update, api::Timestamp(1234))); } void ChangedBucketOwnershipHandlerTest::testAbortOutdatedRemoveCommand() { document::DocumentId docId("id:foo:testdoctype1::bar"); - CPPUNIT_ASSERT(changeAbortsMessage<api::RemoveCommand>( - getBucketToAbort(), docId, api::Timestamp(1234))); - CPPUNIT_ASSERT(!changeAbortsMessage<api::RemoveCommand>( - getBucketToAllow(), docId, api::Timestamp(1234))); + CPPUNIT_ASSERT(changeAbortsMessage<api::RemoveCommand>(getBucketToAbort(), docId, api::Timestamp(1234))); + CPPUNIT_ASSERT(!changeAbortsMessage<api::RemoveCommand>(getBucketToAllow(), docId, api::Timestamp(1234))); } void diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index 695ae17c5d4..40d561bd589 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -181,7 +181,7 @@ void DocumentApiConverterTest::testForwardedPut() void DocumentApiConverterTest::testUpdate() { - auto update = std::make_shared<document::DocumentUpdate>(_html_type, defaultDocId); + auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, defaultDocId); documentapi::UpdateDocumentMessage updateMsg(update); updateMsg.setOldTimestamp(1234); updateMsg.setNewTimestamp(5678); @@ -327,19 +327,19 @@ DocumentApiConverterTest::testBatchDocumentUpdate() { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test1")); - auto update = std::make_shared<document::DocumentUpdate>(_html_type, docId); + auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, docId); updates.push_back(update); } { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test2")); - auto update = std::make_shared<document::DocumentUpdate>(_html_type, docId); + auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, docId); updates.push_back(update); } { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test3")); - auto update = std::make_shared<document::DocumentUpdate>(_html_type, docId); + auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, docId); updates.push_back(update); } diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index e29540de064..2a2a840dd4e 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -28,3 +28,16 @@ mbus.compress.type enum {NONE, LZ4, ZSTD} default=LZ4 ## TTL for rpc target cache mbus.rpctargetcache.ttl double default = 600 + +## Number of threads for mbus threadpool +## Any value below 1 will be 1. +mbus.num_threads int default=4 + +## Enable to use above thread pool for encoding replies +## False will use network(fnet) thread +mbus.dispatch_on_encode bool default=true + +## Enable to use above thread pool for decoding replies +## False will use network(fnet) thread +## Todo: Change default once verified in large scale deployment. +mbus.dispatch_on_decode bool default=false diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 94a151bcdc1..65523b62c59 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -79,7 +79,7 @@ StorageTransportContext::StorageTransportContext(std::unique_ptr<RPCRequestWrapp : _request(std::move(request)) { } -StorageTransportContext::~StorageTransportContext() { } +StorageTransportContext::~StorageTransportContext() = default; void CommunicationManager::receiveStorageReply(const std::shared_ptr<api::StorageReply>& reply) @@ -278,13 +278,13 @@ void CommunicationManager::fail_with_unresolvable_bucket_space( namespace { struct PlaceHolderBucketResolver : public BucketResolver { - virtual document::Bucket bucketFromId(const document::DocumentId &) const override { + document::Bucket bucketFromId(const document::DocumentId &) const override { return document::Bucket(FixedBucketSpaces::default_space(), document::BucketId(0)); } - virtual document::BucketSpace bucketSpaceFromName(const vespalib::string &) const override { + document::BucketSpace bucketSpaceFromName(const vespalib::string &) const override { return FixedBucketSpaces::default_space(); } - virtual vespalib::string nameFromBucketSpace(const document::BucketSpace &bucketSpace) const override { + vespalib::string nameFromBucketSpace(const document::BucketSpace &bucketSpace) const override { assert(bucketSpace == FixedBucketSpaces::default_space()); return FixedBucketSpaces::to_string(bucketSpace); } @@ -423,6 +423,9 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> params.setSlobrokConfig(_configUri); params.setConnectionExpireSecs(config->mbus.rpctargetcache.ttl); + params.setNumThreads(std::max(1, config->mbus.numThreads)); + params.setDispatchOnDecode(config->mbus.dispatchOnDecode); + params.setDispatchOnEncode(config->mbus.dispatchOnEncode); params.setIdentity(mbus::Identity(_component.getIdentity())); if (config->mbusport != -1) { diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index 6bb2ca31ec1..0602d14e088 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -16,6 +16,8 @@ #include <fcntl.h> #include <vespa/log/log.h> +#include <vespa/config/print/configdatabuffer.h> + LOG_SETUP(".node.server"); using vespa::config::content::StorDistributionConfigBuilder; @@ -466,12 +468,24 @@ StorageNode::shutdown() LOG(debug, "Done shutting down node"); } -void StorageNode::configure(std::unique_ptr<StorServerConfig> config) -{ - // When we get config, we try to grab the config lock to ensure noone - // else is doing configuration work, and then we write the new config - // to a variable where we can find it later when processing config - // updates +namespace { + +void log_config_received(const config::ConfigInstance& cfg) { + if (LOG_WOULD_LOG(debug)) { + config::ConfigDataBuffer buf; + cfg.serialize(buf); + LOG(debug, "Received new %s config: %s", cfg.defName().c_str(), buf.getEncodedString().c_str()); + } +} + +} + +void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { + log_config_received(*config); + // When we get config, we try to grab the config lock to ensure noone + // else is doing configuration work, and then we write the new config + // to a variable where we can find it later when processing config + // updates { vespalib::LockGuard configLockGuard(_configLock); _newServerConfig = std::move(config); @@ -482,13 +496,8 @@ void StorageNode::configure(std::unique_ptr<StorServerConfig> config) } } -void -StorageNode::configure(std::unique_ptr<UpgradingConfig> config) -{ - // When we get config, we try to grab the config lock to ensure noone - // else is doing configuration work, and then we write the new config - // to a variable where we can find it later when processing config - // updates +void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { + log_config_received(*config); { vespalib::LockGuard configLockGuard(_configLock); _newClusterConfig = std::move(config); @@ -499,13 +508,8 @@ StorageNode::configure(std::unique_ptr<UpgradingConfig> config) } } -void -StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) -{ - // When we get config, we try to grab the config lock to ensure noone - // else is doing configuration work, and then we write the new config - // to a variable where we can find it later when processing config - // updates +void StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { + log_config_received(*config); { vespalib::LockGuard configLockGuard(_configLock); _newDistributionConfig = std::move(config); @@ -516,9 +520,8 @@ StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) } } -void -StorageNode::configure(std::unique_ptr<StorPrioritymappingConfig> config) -{ +void StorageNode::configure(std::unique_ptr<StorPrioritymappingConfig> config) { + log_config_received(*config); { vespalib::LockGuard configLockGuard(_configLock); _newPriorityConfig = std::move(config); @@ -533,6 +536,7 @@ void StorageNode::configure(std::unique_ptr<document::DocumenttypesConfig> config, bool hasChanged, int64_t generation) { + log_config_received(*config); (void) generation; if (!hasChanged) return; @@ -546,9 +550,8 @@ StorageNode::configure(std::unique_ptr<document::DocumenttypesConfig> config, } } -void -StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) -{ +void StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { + log_config_received(*config); { vespalib::LockGuard configLockGuard(_configLock); _newBucketSpacesConfig = std::move(config); diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index a809d80e0f5..cfc69bcde45 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -294,7 +294,7 @@ void StorageProtocolTest::testUpdate51() { ScopedName test("testUpdate51"); - document::DocumentUpdate::SP update(new document::DocumentUpdate(*_testDoc->getDataType(), _testDoc->getId())); + document::DocumentUpdate::SP update(new document::DocumentUpdate(_docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId())); std::shared_ptr<document::AssignValueUpdate> assignUpdate(new document::AssignValueUpdate(document::IntFieldValue(17))); document::FieldUpdate fieldUpdate(_testDoc->getField("headerval")); fieldUpdate.addUpdate(*assignUpdate); @@ -912,7 +912,7 @@ StorageProtocolTest::testUpdateCommand52() { ScopedName test("testUpdateCommand52"); - document::DocumentUpdate::SP update(new document::DocumentUpdate(*_testDoc->getDataType(), _testDoc->getId())); + document::DocumentUpdate::SP update(new document::DocumentUpdate(_docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId())); UpdateCommand::SP cmd(new UpdateCommand(_bucket, update, 14)); cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp index 44050264484..47b77af74a5 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp @@ -228,9 +228,8 @@ ProtocolSerialization5_0::onDecodeUpdateCommand(BBuf& buf) const uint32_t size = SH::getInt(buf); if (size != 0) { - document::ByteBuffer bbuf(buf.getBufferAtPos(), size); + update = document::DocumentUpdate::createHEAD(getTypeRepo(), vespalib::nbostream(buf.getBufferAtPos(), size)); buf.incPos(size); - update = document::DocumentUpdate::createHEAD(getTypeRepo(), bbuf); } document::Bucket bucket = getBucket(buf); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java index b011cc8fbcc..3a81e4a5e17 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java @@ -15,6 +15,11 @@ public class AthenzRole { this.roleName = roleName; } + public AthenzRole(String domain, String roleName) { + this.domain = new AthenzDomain(domain); + this.roleName = roleName; + } + public AthenzDomain domain() { return domain; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index dfe49d1b407..f8654bbaa68 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -10,12 +10,15 @@ import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.athenz.api.ZToken; +import com.yahoo.vespa.athenz.client.zts.bindings.IdentityResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.InstanceIdentityCredentials; import com.yahoo.vespa.athenz.client.zts.bindings.InstanceRefreshInformation; +import com.yahoo.vespa.athenz.client.zts.bindings.IdentityRefreshRequestEntity; import com.yahoo.vespa.athenz.client.zts.bindings.InstanceRegisterInformation; import com.yahoo.vespa.athenz.client.zts.bindings.RoleCertificateRequestEntity; import com.yahoo.vespa.athenz.client.zts.bindings.RoleCertificateResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.RoleTokenResponseEntity; +import com.yahoo.vespa.athenz.client.zts.utils.IdentityCsrGenerator; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.Pkcs10Csr; import com.yahoo.vespa.athenz.tls.Pkcs10CsrBuilder; @@ -127,12 +130,37 @@ public class DefaultZtsClient implements ZtsClient { } @Override + public Identity getServiceIdentity(AthenzService identity, String keyId, Pkcs10Csr csr) { + URI uri = ztsUrl.resolve(String.format("instance/%s/%s/refresh", identity.getDomainName(), identity.getName())); + HttpUriRequest request = RequestBuilder.post() + .setUri(uri) + .setEntity(toJsonStringEntity(new IdentityRefreshRequestEntity(csr, keyId))) + .build(); + return withClient(client -> { + try (CloseableHttpResponse response = client.execute(request)) { + IdentityResponseEntity entity = readEntity(response, IdentityResponseEntity.class); + return new Identity(entity.certificate(), entity.caCertificateBundle()); + } + }); + } + + @Override + public Identity getServiceIdentity(AthenzService identity, String keyId, KeyPair keyPair, String dnsSuffix) { + Pkcs10Csr csr = new IdentityCsrGenerator(dnsSuffix).generateIdentityCsr(identity, keyPair); + return getServiceIdentity(identity, keyId, csr); + } + + @Override public ZToken getRoleToken(AthenzDomain domain) { return getRoleToken(domain, null); } @Override - public ZToken getRoleToken(AthenzDomain domain, String roleName) { + public ZToken getRoleToken(AthenzRole athenzRole) { + return getRoleToken(athenzRole.domain(), athenzRole.roleName()); + } + + private ZToken getRoleToken(AthenzDomain domain, String roleName) { URI uri = ztsUrl.resolve(String.format("domain/%s/token", domain.getName())); RequestBuilder requestBuilder = RequestBuilder.get(uri) .addHeader("Content-Type", "application/json"); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/Identity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/Identity.java new file mode 100644 index 00000000000..455f3c06d1d --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/Identity.java @@ -0,0 +1,29 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zts; + +import java.security.cert.X509Certificate; +import java.util.List; + +/** + * The identity of a service + * + * @author bjorncs + */ +public class Identity { + + private final X509Certificate certificate; + private final List<X509Certificate> caCertificates; + + public Identity(X509Certificate certificate, List<X509Certificate> caCertificates) { + this.certificate = certificate; + this.caCertificates = caCertificates; + } + + public X509Certificate certificate() { + return certificate; + } + + public List<X509Certificate> caCertificates() { + return caCertificates; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java index 43378b6507a..9502deca1c0 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java @@ -43,6 +43,25 @@ public interface ZtsClient extends AutoCloseable { Pkcs10Csr csr); /** + * Get service identity + * + * @return A x509 certificate with CA certificates + */ + Identity getServiceIdentity(AthenzService identity, + String keyId, + Pkcs10Csr csr); + + /** + * Get service identity + * + * @return A x509 certificate with CA certificates + */ + Identity getServiceIdentity(AthenzService identity, + String keyId, + KeyPair keyPair, + String dnsSuffix); + + /** * Fetch a role token for the target domain * * @param domain Target domain @@ -51,13 +70,12 @@ public interface ZtsClient extends AutoCloseable { ZToken getRoleToken(AthenzDomain domain); /** - * Fetch a role token for the target domain and role + * Fetch a role token for the target role * - * @param domain Target domain - * @param roleName Target role + * @param athenzRole Target role * @return A role token */ - ZToken getRoleToken(AthenzDomain domain, String roleName); + ZToken getRoleToken(AthenzRole athenzRole); /** * Fetch role certificate for the target domain and role diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityRefreshRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityRefreshRequestEntity.java new file mode 100644 index 00000000000..47ae9cd2d3f --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityRefreshRequestEntity.java @@ -0,0 +1,24 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zts.bindings; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.yahoo.vespa.athenz.client.zts.bindings.serializers.Pkcs10CsrSerializer; +import com.yahoo.vespa.athenz.tls.Pkcs10Csr; + +/** + * @author bjorncs + */ +public class IdentityRefreshRequestEntity { + + @JsonProperty("csr") @JsonSerialize(using = Pkcs10CsrSerializer.class) + private final Pkcs10Csr csr; + + @JsonProperty("keyId") + private final String keyId; + + public IdentityRefreshRequestEntity(Pkcs10Csr csr, String keyId) { + this.csr = csr; + this.keyId = keyId; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityResponseEntity.java new file mode 100644 index 00000000000..7bd04362599 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityResponseEntity.java @@ -0,0 +1,40 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zts.bindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.yahoo.vespa.athenz.client.zts.bindings.serializers.X509CertificateDeserializer; +import com.yahoo.vespa.athenz.client.zts.bindings.serializers.X509CertificateListDeserializer; + +import java.security.cert.X509Certificate; +import java.util.List; + +/** + * Identity response entity + * + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class IdentityResponseEntity { + + private final X509Certificate certificate; + private final List<X509Certificate> caCertificateBundle; + + @JsonCreator + public IdentityResponseEntity( + @JsonProperty("certificate") @JsonDeserialize(using = X509CertificateDeserializer.class) X509Certificate certificate, + @JsonProperty("caCertBundle") @JsonDeserialize(using = X509CertificateListDeserializer.class) List<X509Certificate> caCertificateBundle) { + this.certificate = certificate; + this.caCertificateBundle = caCertificateBundle; + } + + public X509Certificate certificate() { + return certificate; + } + + public List<X509Certificate> caCertificateBundle() { + return caCertificateBundle; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceIdentityCredentials.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceIdentityCredentials.java index 5c265f14813..0ab697a1c4c 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceIdentityCredentials.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceIdentityCredentials.java @@ -4,13 +4,9 @@ package com.yahoo.vespa.athenz.client.zts.bindings; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.yahoo.vespa.athenz.tls.X509CertificateUtils; +import com.yahoo.vespa.athenz.client.zts.bindings.serializers.X509CertificateDeserializer; -import java.io.IOException; import java.security.cert.X509Certificate; /** @@ -39,11 +35,4 @@ public class InstanceIdentityCredentials { return serviceToken; } - public static class X509CertificateDeserializer extends JsonDeserializer<X509Certificate> { - @Override - public X509Certificate deserialize(JsonParser parser, DeserializationContext context) throws IOException { - return X509CertificateUtils.fromPem(parser.getValueAsString()); - } - } - } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRefreshInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRefreshInformation.java index 6c956ddb410..b842ef43500 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRefreshInformation.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRefreshInformation.java @@ -4,8 +4,9 @@ package com.yahoo.vespa.athenz.client.zts.bindings; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.yahoo.vespa.athenz.client.zts.bindings.serializers.Pkcs10CsrSerializer; import com.yahoo.vespa.athenz.tls.Pkcs10Csr; -import com.yahoo.vespa.athenz.tls.Pkcs10CsrUtils; /** * @author bjorncs @@ -15,13 +16,14 @@ import com.yahoo.vespa.athenz.tls.Pkcs10CsrUtils; public class InstanceRefreshInformation { @JsonProperty("csr") - private final String csr; + @JsonSerialize(using = Pkcs10CsrSerializer.class) + private final Pkcs10Csr csr; @JsonProperty("token") private final boolean requestServiceToken; public InstanceRefreshInformation(Pkcs10Csr csr, boolean requestServiceToken) { - this.csr = Pkcs10CsrUtils.toPem(csr); + this.csr = csr; this.requestServiceToken = requestServiceToken; } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java index 9c56e5a60d6..f329ffbbd2c 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java @@ -8,8 +8,8 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.yahoo.vespa.athenz.client.zts.bindings.serializers.Pkcs10CsrSerializer; import com.yahoo.vespa.athenz.tls.Pkcs10Csr; -import com.yahoo.vespa.athenz.tls.Pkcs10CsrUtils; import java.io.IOException; import java.time.Duration; @@ -20,7 +20,7 @@ import java.time.Duration; @JsonIgnoreProperties(ignoreUnknown = true) public class RoleCertificateRequestEntity { @JsonProperty("csr") - @JsonSerialize(using = CsrSerializer.class) + @JsonSerialize(using = Pkcs10CsrSerializer.class) public final Pkcs10Csr csr; @JsonProperty("expiryTime") @@ -33,15 +33,6 @@ public class RoleCertificateRequestEntity { this.expiryTime = expiryTime; } - public static class CsrSerializer extends JsonSerializer<Pkcs10Csr> { - @Override - public void serialize(Pkcs10Csr csr, - JsonGenerator jsonGenerator, - SerializerProvider serializerProvider) throws IOException { - jsonGenerator.writeString(Pkcs10CsrUtils.toPem(csr)); - } - } - public static class ExpirySerializer extends JsonSerializer<Duration> { @Override public void serialize(Duration duration, diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java index 1b4bd463392..e80f5626843 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java @@ -4,13 +4,9 @@ package com.yahoo.vespa.athenz.client.zts.bindings; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.yahoo.vespa.athenz.tls.X509CertificateUtils; +import com.yahoo.vespa.athenz.client.zts.bindings.serializers.X509CertificateDeserializer; -import java.io.IOException; import java.security.cert.X509Certificate; import java.time.Instant; @@ -28,11 +24,4 @@ public class RoleCertificateResponseEntity { this.certificate = certificate; this.expiry = expiry; } - - public static class X509CertificateDeserializer extends JsonDeserializer<X509Certificate> { - @Override - public X509Certificate deserialize(JsonParser parser, DeserializationContext context) throws IOException { - return X509CertificateUtils.fromPem(parser.getValueAsString()); - } - } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/Pkcs10CsrSerializer.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/Pkcs10CsrSerializer.java new file mode 100644 index 00000000000..24825792953 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/Pkcs10CsrSerializer.java @@ -0,0 +1,20 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zts.bindings.serializers; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.yahoo.vespa.athenz.tls.Pkcs10Csr; +import com.yahoo.vespa.athenz.tls.Pkcs10CsrUtils; + +import java.io.IOException; + +/** + * @author bjorncs + */ +public class Pkcs10CsrSerializer extends JsonSerializer<Pkcs10Csr> { + @Override + public void serialize(Pkcs10Csr csr, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeString(Pkcs10CsrUtils.toPem(csr)); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/X509CertificateDeserializer.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/X509CertificateDeserializer.java new file mode 100644 index 00000000000..5dd6ceb16b4 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/X509CertificateDeserializer.java @@ -0,0 +1,21 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zts.bindings.serializers; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.yahoo.vespa.athenz.tls.X509CertificateUtils; + +import java.io.IOException; +import java.security.cert.X509Certificate; + +/** + * @author bjorncs + */ +public class X509CertificateDeserializer extends JsonDeserializer<X509Certificate> { + @Override + public X509Certificate deserialize(JsonParser parser, DeserializationContext context) throws IOException { + return X509CertificateUtils.fromPem(parser.getValueAsString()); + } +} + diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/X509CertificateListDeserializer.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/X509CertificateListDeserializer.java new file mode 100644 index 00000000000..c496031c116 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/X509CertificateListDeserializer.java @@ -0,0 +1,22 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zts.bindings.serializers; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.yahoo.vespa.athenz.tls.X509CertificateUtils; + +import java.io.IOException; +import java.security.cert.X509Certificate; +import java.util.List; + +/** + * @author bjorncs + */ +public class X509CertificateListDeserializer extends JsonDeserializer<List<X509Certificate>> { + + @Override + public List<X509Certificate> deserialize(JsonParser parser, DeserializationContext ctxt) throws IOException { + return X509CertificateUtils.certificateListFromPem(parser.getValueAsString()); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/package-info.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/package-info.java new file mode 100644 index 00000000000..4c442617494 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.vespa.athenz.client.zts.bindings.serializers; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java new file mode 100644 index 00000000000..2f152fafba8 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java @@ -0,0 +1,37 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zts.utils; + +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.client.zts.ZtsClient; +import com.yahoo.vespa.athenz.tls.Pkcs10Csr; +import com.yahoo.vespa.athenz.tls.Pkcs10CsrBuilder; + +import javax.security.auth.x500.X500Principal; +import java.security.KeyPair; + +import static com.yahoo.vespa.athenz.tls.SignatureAlgorithm.SHA256_WITH_RSA; + +/** + * Generates a {@link Pkcs10Csr} instance for use with {@link ZtsClient#getServiceIdentity(AthenzService, String, Pkcs10Csr)} + * + * @author bjorncs + */ +public class IdentityCsrGenerator { + + private final String dnsSuffix; + + public IdentityCsrGenerator(String dnsSuffix) { + this.dnsSuffix = dnsSuffix; + } + + public Pkcs10Csr generateIdentityCsr(AthenzService identity, KeyPair keypair) { + return Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=" + identity.getFullName()), keypair, SHA256_WITH_RSA) + .addSubjectAlternativeName(String.format( + "%s.%s.%s", + identity.getName(), + identity.getDomainName().replace(".", "-"), + dnsSuffix)) + .build(); + } + +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/package-info.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/package-info.java new file mode 100644 index 00000000000..baca71bc187 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.vespa.athenz.client.zts.utils; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index ab127b19bf1..3bfe492a125 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentEntity; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.identityprovider.api.bindings.VespaUniqueInstanceIdEntity; -import com.yahoo.vespa.athenz.utils.AthenzIdentities; import java.io.IOException; import java.io.UncheckedIOException; @@ -52,12 +51,12 @@ public class EntityBindingsMapper { public static SignedIdentityDocument toSignedIdentityDocument(SignedIdentityDocumentEntity entity) { return new SignedIdentityDocument( - toIdentityDocument(entity.identityDocument), + entity.identityDocument != null ? toIdentityDocument(entity.identityDocument) : null, entity.signature, entity.signingKeyVersion, fromDottedString(entity.providerUniqueId), entity.dnsSuffix, - (AthenzService) AthenzIdentities.from(entity.providerService), + new AthenzService(entity.providerService), entity.ztsEndpoint, entity.documentVersion, entity.configServerHostname, @@ -84,7 +83,7 @@ public class EntityBindingsMapper { public static SignedIdentityDocumentEntity toSignedIdentityDocumentEntity(SignedIdentityDocument model) { try { - IdentityDocumentEntity identityDocumentEntity = toIdentityDocumentEntity(model.identityDocument()); + IdentityDocumentEntity identityDocumentEntity = model.identityDocument() != null ? toIdentityDocumentEntity(model.identityDocument()) : null; String rawDocument = Base64.getEncoder().encodeToString(mapper.writeValueAsString(identityDocumentEntity).getBytes()); return new SignedIdentityDocumentEntity( rawDocument, diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index 7c64d048944..aa1dbd4dac3 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -58,6 +58,7 @@ public class SignedIdentityDocument { this.identityType = identityType; } + @Deprecated public IdentityDocument identityDocument() { return identityDocument; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index aa514b3caf3..3d4872549d6 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -25,7 +25,7 @@ public class SignedIdentityDocumentEntity { private static final ObjectMapper mapper = createObjectMapper(); @JsonProperty("identity-document")public final String rawIdentityDocument; - @JsonIgnore public final IdentityDocumentEntity identityDocument; + @JsonIgnore @Deprecated public final IdentityDocumentEntity identityDocument; @JsonProperty("signature") public final String signature; @JsonProperty("signing-key-version") public final int signingKeyVersion; @JsonProperty("provider-unique-id") public final String providerUniqueId; // String representation diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java index bb9f512efe6..a1d8a9ca258 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java @@ -12,28 +12,21 @@ import java.security.cert.X509Certificate; */ class AthenzCredentials { - private final String nToken; private final X509Certificate certificate; private final KeyPair keyPair; private final SignedIdentityDocument identityDocument; private final SSLContext identitySslContext; - AthenzCredentials(String nToken, - X509Certificate certificate, + AthenzCredentials(X509Certificate certificate, KeyPair keyPair, SignedIdentityDocument identityDocument, SSLContext identitySslContext) { - this.nToken = nToken; this.certificate = certificate; this.keyPair = keyPair; this.identityDocument = identityDocument; this.identitySslContext = identitySslContext; } - String getNToken() { - return nToken; - } - X509Certificate getCertificate() { return certificate; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java index 1136106ce19..5567831d49d 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java @@ -5,6 +5,7 @@ import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; +import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; @@ -14,13 +15,21 @@ import com.yahoo.vespa.athenz.tls.KeyAlgorithm; import com.yahoo.vespa.athenz.tls.KeyUtils; import com.yahoo.vespa.athenz.tls.Pkcs10Csr; import com.yahoo.vespa.athenz.tls.SslContextBuilder; +import com.yahoo.vespa.athenz.utils.SiaUtils; +import com.yahoo.vespa.defaults.Defaults; import javax.net.ssl.SSLContext; import java.io.File; import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.KeyPair; import java.security.PrivateKey; import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Duration; +import java.util.Optional; import static com.yahoo.vespa.athenz.tls.KeyStoreType.JKS; import static java.util.Collections.singleton; @@ -31,77 +40,114 @@ import static java.util.Collections.singleton; * @author bjorncs */ class AthenzCredentialsService { - private final IdentityConfig identityConfig; + private static final Duration EXPIRATION_MARGIN = Duration.ofDays(2); + private static final Path VESPA_SIA_DIRECTORY = Paths.get(Defaults.getDefaults().underVespaHome("var/vespa/sia")); + private static final Path IDENTITY_DOCUMENT_FILE = VESPA_SIA_DIRECTORY.resolve("vespa-tenant-identity-document.json"); + + private final AthenzService tenantIdentity; + private final URI configserverEndpoint; + private final URI ztsEndpoint; + private final AthenzService configserverIdentity; private final ServiceIdentityProvider nodeIdentityProvider; private final File trustStoreJks; private final String hostname; private final InstanceCsrGenerator instanceCsrGenerator; + private final Clock clock; AthenzCredentialsService(IdentityConfig identityConfig, ServiceIdentityProvider nodeIdentityProvider, File trustStoreJks, - String hostname) { - this.identityConfig = identityConfig; + String hostname, + Clock clock) { + this.tenantIdentity = new AthenzService(identityConfig.domain(), identityConfig.service()); + this.configserverEndpoint = URI.create("https://" + identityConfig.loadBalancerAddress() + ":4443"); + this.ztsEndpoint = URI.create(identityConfig.ztsUrl()); + this.configserverIdentity = new AthenzService(identityConfig.configserverIdentityName()); this.nodeIdentityProvider = nodeIdentityProvider; this.trustStoreJks = trustStoreJks; this.hostname = hostname; this.instanceCsrGenerator = new InstanceCsrGenerator(identityConfig.athenzDnsSuffix()); + this.clock = clock; } AthenzCredentials registerInstance() { + Optional<AthenzCredentials> athenzCredentialsFromDisk = tryReadCredentialsFromDisk(); + if (athenzCredentialsFromDisk.isPresent()) { + return athenzCredentialsFromDisk.get(); + } KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - IdentityDocumentClient identityDocumentClient = createIdentityDocumentClient(identityConfig, nodeIdentityProvider); + IdentityDocumentClient identityDocumentClient = createIdentityDocumentClient(); SignedIdentityDocument document = identityDocumentClient.getTenantIdentityDocument(hostname); - AthenzService tenantIdentity = new AthenzService(identityConfig.domain(), identityConfig.service()); Pkcs10Csr csr = instanceCsrGenerator.generateCsr( tenantIdentity, document.providerUniqueId(), - document.identityDocument().ipAddresses(), + document.ipAddresses(), keyPair); - try (com.yahoo.vespa.athenz.client.zts.ZtsClient ztsClient = - new DefaultZtsClient(URI.create(identityConfig.ztsUrl()), nodeIdentityProvider)) { + try (ZtsClient ztsClient = new DefaultZtsClient(ztsEndpoint, nodeIdentityProvider)) { InstanceIdentity instanceIdentity = ztsClient.registerInstance( - new AthenzService(identityConfig.configserverIdentityName()), + configserverIdentity, tenantIdentity, null, EntityBindingsMapper.toAttestationData(document), - true, + false, csr); - return toAthenzCredentials(instanceIdentity, keyPair, document); + X509Certificate certificate = instanceIdentity.certificate(); + SSLContext identitySslContext = createIdentitySslContext(keyPair.getPrivate(), certificate); + writeCredentialsToDisk(keyPair.getPrivate(), certificate, document); + return new AthenzCredentials(certificate, keyPair, document, identitySslContext); } } AthenzCredentials updateCredentials(SignedIdentityDocument document, SSLContext sslContext) { - AthenzService tenantIdentity = new AthenzService(identityConfig.domain(), identityConfig.service()); KeyPair newKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); Pkcs10Csr csr = instanceCsrGenerator.generateCsr( tenantIdentity, document.providerUniqueId(), - document.identityDocument().ipAddresses(), + document.ipAddresses(), newKeyPair); - try (com.yahoo.vespa.athenz.client.zts.ZtsClient ztsClient = - new DefaultZtsClient(URI.create(identityConfig.ztsUrl()), tenantIdentity, sslContext)) { + try (ZtsClient ztsClient = new DefaultZtsClient(ztsEndpoint, tenantIdentity, sslContext)) { InstanceIdentity instanceIdentity = ztsClient.refreshInstance( - new AthenzService(identityConfig.configserverIdentityName()), + configserverIdentity, tenantIdentity, document.providerUniqueId().asDottedString(), - true, + false, csr); - return toAthenzCredentials(instanceIdentity, newKeyPair, document); + X509Certificate certificate = instanceIdentity.certificate(); + SSLContext identitySslContext = createIdentitySslContext(newKeyPair.getPrivate(), certificate); + writeCredentialsToDisk(newKeyPair.getPrivate(), certificate, document); + return new AthenzCredentials(certificate, newKeyPair, document, identitySslContext); } } - private AthenzCredentials toAthenzCredentials(InstanceIdentity instanceIdentity, - KeyPair keyPair, - SignedIdentityDocument identityDocument) { - X509Certificate certificate = instanceIdentity.certificate(); - String serviceToken = instanceIdentity.nToken().get().getRawToken(); - SSLContext identitySslContext = createIdentitySslContext(keyPair.getPrivate(), certificate); - return new AthenzCredentials(serviceToken, certificate, keyPair, identityDocument, identitySslContext); + private Optional<AthenzCredentials> tryReadCredentialsFromDisk() { + Optional<PrivateKey> privateKey = SiaUtils.readPrivateKeyFile(VESPA_SIA_DIRECTORY, tenantIdentity); + if (!privateKey.isPresent()) return Optional.empty(); + Optional<X509Certificate> certificate = SiaUtils.readCertificateFile(VESPA_SIA_DIRECTORY, tenantIdentity); + if (!certificate.isPresent()) return Optional.empty(); + if (isExpired(certificate.get())) { + return Optional.empty(); + } + if (Files.notExists(IDENTITY_DOCUMENT_FILE)) return Optional.empty(); + SignedIdentityDocument signedIdentityDocument = EntityBindingsMapper.readSignedIdentityDocumentFromFile(IDENTITY_DOCUMENT_FILE); + KeyPair keyPair = new KeyPair(KeyUtils.extractPublicKey(privateKey.get()), privateKey.get()); + SSLContext sslContext = createIdentitySslContext(privateKey.get(), certificate.get()); + return Optional.of(new AthenzCredentials(certificate.get(), keyPair, signedIdentityDocument, sslContext)); + } + + private boolean isExpired(X509Certificate certificate) { + return clock.instant().isAfter(certificate.getNotAfter().toInstant().minus(EXPIRATION_MARGIN)); + } + + private void writeCredentialsToDisk(PrivateKey privateKey, + X509Certificate certificate, + SignedIdentityDocument identityDocument) { + SiaUtils.writePrivateKeyFile(VESPA_SIA_DIRECTORY, tenantIdentity, privateKey); + SiaUtils.writeCertificateFile(VESPA_SIA_DIRECTORY, tenantIdentity, certificate); + EntityBindingsMapper.writeSignedIdentityDocumentToFile(IDENTITY_DOCUMENT_FILE, identityDocument); } private SSLContext createIdentitySslContext(PrivateKey privateKey, X509Certificate certificate) { @@ -111,11 +157,10 @@ class AthenzCredentialsService { .build(); } - private static DefaultIdentityDocumentClient createIdentityDocumentClient(IdentityConfig config, - ServiceIdentityProvider nodeIdentityProvider) { + private DefaultIdentityDocumentClient createIdentityDocumentClient() { return new DefaultIdentityDocumentClient( - URI.create(config.loadBalancerAddress()), + configserverEndpoint, nodeIdentityProvider, - new AthenzIdentityVerifier(singleton(new AthenzService(config.configserverIdentityName())))); + new AthenzIdentityVerifier(singleton(configserverIdentity))); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index ce0743021ff..e40a0933002 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -14,6 +14,9 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.ZToken; +import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; +import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identity.ServiceIdentityProviderListenerHelper; import com.yahoo.vespa.athenz.identity.SiaIdentityProvider; @@ -25,7 +28,6 @@ import com.yahoo.vespa.defaults.Defaults; import javax.net.ssl.SSLContext; import java.io.File; import java.net.URI; -import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; @@ -33,9 +35,12 @@ import java.time.Instant; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.logging.Logger; /** + * A {@link AthenzIdentityProvider} / {@link ServiceIdentityProvider} component that provides the tenant identity. + * * @author mortent * @author bjorncs */ @@ -47,11 +52,12 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen // TODO These should match the requested expiration static final Duration UPDATE_PERIOD = Duration.ofDays(1); static final Duration AWAIT_TERMINTATION_TIMEOUT = Duration.ofSeconds(90); + private final static Duration ROLE_SSL_CONTEXT_EXPIRY = Duration.ofHours(24); + private final static Duration ROLE_TOKEN_EXPIRY = Duration.ofMinutes(30); public static final String CERTIFICATE_EXPIRY_METRIC_NAME = "athenz-tenant-cert.expiry.seconds"; private volatile AthenzCredentials credentials; - private final ZtsClient ztsClient = new ZtsClient(); private final Metric metric; private final AthenzCredentialsService athenzCredentialsService; private final ScheduledExecutorService scheduler; @@ -62,7 +68,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private final URI ztsEndpoint; private final LoadingCache<AthenzRole, SSLContext> roleSslContextCache; - private final static Duration roleSslContextExpiry = Duration.ofHours(24); + private final LoadingCache<AthenzRole, ZToken> roleSpecificRoleTokenCache; + private final LoadingCache<AthenzDomain, ZToken> domainSpecificRoleTokenCache; @Inject public AthenzIdentityProviderImpl(IdentityConfig config, Metric metric) { @@ -71,7 +78,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen new AthenzCredentialsService(config, createNodeIdentityProvider(config), getDefaultTrustStoreLocation(), - Defaults.getDefaults().vespaHostname()), + Defaults.getDefaults().vespaHostname(), + Clock.systemUTC()), new ScheduledThreadPoolExecutor(1), Clock.systemUTC()); } @@ -91,14 +99,20 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen this.listenerHelper = new ServiceIdentityProviderListenerHelper(this.identity); this.dnsSuffix = config.athenzDnsSuffix(); this.ztsEndpoint = URI.create(config.ztsUrl()); + roleSslContextCache = createCache(ROLE_SSL_CONTEXT_EXPIRY, this::createRoleSslContext); + roleSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); + domainSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); registerInstance(); - roleSslContextCache = CacheBuilder.newBuilder() - .refreshAfterWrite(roleSslContextExpiry.dividedBy(2).toMinutes(), TimeUnit.MINUTES) - .expireAfterWrite(roleSslContextExpiry.toMinutes(), TimeUnit.MINUTES) - .build(new CacheLoader<AthenzRole, SSLContext>() { + } + + private static <KEY, VALUE> LoadingCache<KEY, VALUE> createCache(Duration expiry, Function<KEY, VALUE> cacheLoader) { + return CacheBuilder.newBuilder() + .refreshAfterWrite(expiry.dividedBy(2).toMinutes(), TimeUnit.MINUTES) + .expireAfterWrite(expiry.toMinutes(), TimeUnit.MINUTES) + .build(new CacheLoader<KEY, VALUE>() { @Override - public SSLContext load(AthenzRole key) throws Exception { - return createRoleSslContext(key); + public VALUE load(KEY key) { + return cacheLoader.apply(key); } }); } @@ -149,44 +163,52 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen try { return roleSslContextCache.get(new AthenzRole(new AthenzDomain(domain), role)); } catch (Exception e) { - throw new AthenzIdentityProviderException("Could not retrieve role certificate.", e); + throw new AthenzIdentityProviderException("Could not retrieve role certificate: " + e.getMessage(), e); } } - private SSLContext createRoleSslContext(AthenzRole role) { - PrivateKey privateKey = credentials.getKeyPair().getPrivate(); - X509Certificate roleCertificate = ztsClient.getRoleCertificate( - role, - dnsSuffix, - ztsEndpoint, - identity, - privateKey, - credentials.getIdentitySslContext()); - return new SslContextBuilder() - .withKeyStore(privateKey, roleCertificate) - .withTrustStore(getDefaultTrustStoreLocation(), KeyStoreType.JKS) - .build(); - } - @Override public String getRoleToken(String domain) { - return ztsClient - .getRoleToken( - new AthenzDomain(domain), - ztsEndpoint, - credentials.getIdentitySslContext()) - .getRawToken(); + try { + return domainSpecificRoleTokenCache.get(new AthenzDomain(domain)).getRawToken(); + } catch (Exception e) { + throw new AthenzIdentityProviderException("Could not retrieve role token: " + e.getMessage(), e); + } } @Override public String getRoleToken(String domain, String role) { - return ztsClient - .getRoleToken( - new AthenzDomain(domain), - role, - ztsEndpoint, - credentials.getIdentitySslContext()) - .getRawToken(); + try { + return roleSpecificRoleTokenCache.get(new AthenzRole(domain, role)).getRawToken(); + } catch (Exception e) { + throw new AthenzIdentityProviderException("Could not retrieve role token: " + e.getMessage(), e); + } + } + + private SSLContext createRoleSslContext(AthenzRole role) { + try (ZtsClient client = createZtsClient()) { + X509Certificate roleCertificate = client.getRoleCertificate(role, credentials.getKeyPair(), dnsSuffix); + return new SslContextBuilder() + .withKeyStore(credentials.getKeyPair().getPrivate(), roleCertificate) + .withTrustStore(getDefaultTrustStoreLocation(), KeyStoreType.JKS) + .build(); + } + } + + private ZToken createRoleToken(AthenzRole athenzRole) { + try (ZtsClient client = createZtsClient()) { + return client.getRoleToken(athenzRole); + } + } + + private ZToken createRoleToken(AthenzDomain domain) { + try (ZtsClient client = createZtsClient()) { + return client.getRoleToken(domain); + } + } + + private DefaultZtsClient createZtsClient() { + return new DefaultZtsClient(ztsEndpoint, identity(), getIdentitySslContext()); } @Override diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java new file mode 100644 index 00000000000..b3b5df0e68b --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java @@ -0,0 +1,85 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.client; + +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; +import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; +import com.yahoo.vespa.athenz.tls.SignatureAlgorithm; + +import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.Signature; +import java.security.SignatureException; +import java.time.Instant; +import java.util.Base64; +import java.util.Set; +import java.util.TreeSet; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Generates and validates the signature for a {@link SignedIdentityDocument} + * + * @author bjorncs + */ +public class IdentityDocumentSigner { + + public String generateSignature(VespaUniqueInstanceId providerUniqueId, + AthenzService providerService, + String configServerHostname, + String instanceHostname, + Instant createdAt, + Set<String> ipAddresses, + IdentityType identityType, + PrivateKey privateKey) { + try { + Signature signer = createSigner(); + signer.initSign(privateKey); + writeToSigner(signer, providerUniqueId, providerService, configServerHostname, instanceHostname, createdAt, ipAddresses, identityType); + byte[] signature = signer.sign(); + return Base64.getEncoder().encodeToString(signature); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + } + + public boolean hasValidSignature(SignedIdentityDocument doc, PublicKey publicKey) { + try { + Signature signer = createSigner(); + signer.initVerify(publicKey); + writeToSigner(signer, doc.providerUniqueId(), doc.providerService(), doc.configServerHostname(), doc.instanceHostname(), doc.createdAt(), doc.ipAddresses(), doc.identityType()); + return signer.verify(Base64.getDecoder().decode(doc.signature())); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + } + + private static Signature createSigner() throws NoSuchAlgorithmException { + return Signature.getInstance(SignatureAlgorithm.SHA512_WITH_RSA.getAlgorithmName()); + } + + private static void writeToSigner(Signature signer, + VespaUniqueInstanceId providerUniqueId, + AthenzService providerService, + String configServerHostname, + String instanceHostname, + Instant createdAt, + Set<String> ipAddresses, + IdentityType identityType) throws SignatureException { + signer.update(providerUniqueId.asDottedString().getBytes(UTF_8)); + signer.update(providerService.getFullName().getBytes(UTF_8)); + signer.update(configServerHostname.getBytes(UTF_8)); + signer.update(instanceHostname.getBytes(UTF_8)); + ByteBuffer timestampAsBuffer = ByteBuffer.allocate(Long.BYTES); + timestampAsBuffer.putLong(createdAt.toEpochMilli()); + signer.update(timestampAsBuffer.array()); + for (String ipAddress : new TreeSet<>(ipAddresses)) { + signer.update(ipAddress.getBytes(UTF_8)); + } + signer.update(identityType.id().getBytes(UTF_8)); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java deleted file mode 100644 index a3ec55eb815..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.client; - -import com.yahoo.athenz.zts.RoleCertificateRequest; -import com.yahoo.athenz.zts.RoleToken; -import com.yahoo.athenz.zts.ZTSClient; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.api.ZToken; -import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.athenz.tls.X509CertificateUtils; - -import javax.net.ssl.SSLContext; -import java.net.URI; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.time.Duration; - -/** - * @author mortent - * @author bjorncs - * @deprecated Will be replaced by {@link DefaultZtsClient} once role token/certificate caching is ready. - */ -@Deprecated -class ZtsClient { - - ZToken getRoleToken(AthenzDomain domain, - URI ztsEndpoint, - SSLContext sslContext) { - // TODO ztsEndpoint should contain '/zts/v1' as path - URI correctedZtsEndpoint = ztsEndpoint.resolve("/zts/v1"); - return new ZToken( - new ZTSClient(correctedZtsEndpoint.toString(), sslContext) - .getRoleToken(domain.getName()).getToken()); - } - - ZToken getRoleToken(AthenzDomain domain, - String roleName, - URI ztsEndpoint, - SSLContext sslContext) { - // TODO ztsEndpoint should contain '/zts/v1' as path - URI correctedZtsEndpoint = ztsEndpoint.resolve("/zts/v1"); - return new ZToken( - new ZTSClient(correctedZtsEndpoint.toString(), sslContext) - .getRoleToken(domain.getName(), roleName).getToken()); - } - - X509Certificate getRoleCertificate(AthenzRole role, - String dnsSuffix, - URI ztsEndpoint, - AthenzService identity, - PrivateKey privateKey, - SSLContext sslContext) { - // TODO ztsEndpoint should contain '/zts/v1' as path - URI correctedZtsEndpoint = ztsEndpoint.resolve("/zts/v1"); - ZTSClient ztsClient = new ZTSClient(correctedZtsEndpoint.toString(), sslContext); - RoleCertificateRequest rcr = ZTSClient.generateRoleCertificateRequest( - identity.getDomain().getName(), identity.getName(), role.domain().getName(), role.roleName(), privateKey, dnsSuffix, (int) Duration.ofHours(1).getSeconds()); - RoleToken pemCert = ztsClient.postRoleCertificateRequest(role.domain().getName(), role.roleName(), rcr); - return X509CertificateUtils.fromPem(pemCert.token); - } - -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/SignatureAlgorithm.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/SignatureAlgorithm.java index 3d85a12f714..2f3e2721751 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/SignatureAlgorithm.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/SignatureAlgorithm.java @@ -5,7 +5,8 @@ package com.yahoo.vespa.athenz.tls; * @author bjorncs */ public enum SignatureAlgorithm { - SHA256_WITH_RSA("SHA256withRSA"); + SHA256_WITH_RSA("SHA256withRSA"), + SHA512_WITH_RSA("SHA512withRSA"); private final String algorithmName; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/X509CertificateUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/X509CertificateUtils.java index 6ba094ff275..d96ed17765c 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/X509CertificateUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/X509CertificateUtils.java @@ -21,6 +21,7 @@ import java.io.UncheckedIOException; import java.security.GeneralSecurityException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -36,16 +37,22 @@ public class X509CertificateUtils { public static X509Certificate fromPem(String pem) { try (PEMParser parser = new PEMParser(new StringReader(pem))) { - Object pemObject = parser.readObject(); - if (pemObject instanceof X509Certificate) { - return (X509Certificate) pemObject; - } - if (pemObject instanceof X509CertificateHolder) { - return new JcaX509CertificateConverter() - .setProvider(BouncyCastleProviderHolder.getInstance()) - .getCertificate((X509CertificateHolder) pemObject); + return toX509Certificate(parser.readObject()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (CertificateException e) { + throw new RuntimeException(e); + } + } + + public static List<X509Certificate> certificateListFromPem(String pem) { + try (PEMParser parser = new PEMParser(new StringReader(pem))) { + List<X509Certificate> list = new ArrayList<>(); + Object pemObject; + while ((pemObject = parser.readObject()) != null) { + list.add(toX509Certificate(pemObject)); } - throw new IllegalArgumentException("Invalid type of PEM object: " + pemObject); + return list; } catch (IOException e) { throw new UncheckedIOException(e); } catch (CertificateException e) { @@ -53,6 +60,18 @@ public class X509CertificateUtils { } } + private static X509Certificate toX509Certificate(Object pemObject) throws CertificateException { + if (pemObject instanceof X509Certificate) { + return (X509Certificate) pemObject; + } + if (pemObject instanceof X509CertificateHolder) { + return new JcaX509CertificateConverter() + .setProvider(BouncyCastleProviderHolder.getInstance()) + .getCertificate((X509CertificateHolder) pemObject); + } + throw new IllegalArgumentException("Invalid type of PEM object: " + pemObject); + } + public static String toPem(X509Certificate certificate) { try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { pemWriter.writeObject(new PemObject("CERTIFICATE", certificate.getEncoded())); @@ -65,6 +84,20 @@ public class X509CertificateUtils { } } + public static String toPem(List<X509Certificate> certificates) { + try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { + for (X509Certificate certificate : certificates) { + pemWriter.writeObject(new PemObject("CERTIFICATE", certificate.getEncoded())); + } + pemWriter.flush(); + return stringWriter.toString(); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public static List<String> getSubjectCommonNames(X509Certificate certificate) { return getCommonNames(certificate.getSubjectX500Principal()); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java index adaafab4617..55e9103b040 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -2,9 +2,18 @@ package com.yahoo.vespa.athenz.utils; import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.tls.KeyUtils; +import com.yahoo.vespa.athenz.tls.X509CertificateUtils; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.Optional; /** * Misc utility methods for SIA provided credentials @@ -36,4 +45,68 @@ public class SiaUtils { .resolve(String.format("%s.%s.cert.pem", service.getDomainName(), service.getName())); } + public static Optional<PrivateKey> readPrivateKeyFile(AthenzService service) { + return readPrivateKeyFile(DEFAULT_SIA_DIRECTORY, service); + } + + public static Optional<PrivateKey> readPrivateKeyFile(Path root, AthenzService service) { + try { + Path privateKeyFile = getPrivateKeyFile(root, service); + if (Files.notExists(privateKeyFile)) return Optional.empty(); + return Optional.of(KeyUtils.fromPemEncodedPrivateKey(new String(Files.readAllBytes(privateKeyFile)))); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static Optional<X509Certificate> readCertificateFile(AthenzService service) { + return readCertificateFile(DEFAULT_SIA_DIRECTORY, service); + } + + public static Optional<X509Certificate> readCertificateFile(Path root, AthenzService service) { + try { + Path certificateFile = getCertificateFile(root, service); + if (Files.notExists(certificateFile)) return Optional.empty(); + return Optional.of(X509CertificateUtils.fromPem(new String(Files.readAllBytes(certificateFile)))); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static void writePrivateKeyFile(AthenzService service, PrivateKey privateKey) { + writePrivateKeyFile(DEFAULT_SIA_DIRECTORY, service, privateKey); + } + + public static void writePrivateKeyFile(Path root, AthenzService service, PrivateKey privateKey) { + try { + Path privateKeyFile = getPrivateKeyFile(root, service); + Files.createDirectories(privateKeyFile.getParent()); + Path tempFile = toTempFile(privateKeyFile); + Files.write(tempFile, KeyUtils.toPem(privateKey).getBytes()); + Files.move(tempFile, privateKeyFile, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static void writeCertificateFile(AthenzService service, X509Certificate certificate) { + writeCertificateFile(DEFAULT_SIA_DIRECTORY, service, certificate); + } + + public static void writeCertificateFile(Path root, AthenzService service, X509Certificate certificate) { + try { + Path certificateFile = getCertificateFile(root, service); + Files.createDirectories(certificateFile.getParent()); + Path tempFile = toTempFile(certificateFile); + Files.write(tempFile, X509CertificateUtils.toPem(certificate).getBytes()); + Files.move(tempFile, certificateFile, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static Path toTempFile(Path file) { + return Paths.get(file.toAbsolutePath().toString() + ".tmp"); + } + } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java index 48781aad651..01dab2dada3 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java @@ -62,12 +62,12 @@ public class AthenzIdentityProviderImplTest { X509Certificate certificate = getCertificate(getExpirationSupplier(clock)); when(athenzCredentialsService.registerInstance()) - .thenReturn(new AthenzCredentials(null, certificate, null, null, null)); + .thenReturn(new AthenzCredentials(certificate, null, null, null)); when(athenzCredentialsService.updateCredentials(any(), any())) .thenThrow(new RuntimeException("#1")) .thenThrow(new RuntimeException("#2")) - .thenReturn(new AthenzCredentials(null, certificate, null, null, null)); + .thenReturn(new AthenzCredentials(certificate, null, null, null)); AthenzIdentityProviderImpl identityProvider = new AthenzIdentityProviderImpl(IDENTITY_CONFIG, metric, athenzCredentialsService, mock(ScheduledExecutorService.class), clock); diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java new file mode 100644 index 00000000000..9cc500ee241 --- /dev/null +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java @@ -0,0 +1,50 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.client; + +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; +import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; +import com.yahoo.vespa.athenz.tls.KeyAlgorithm; +import com.yahoo.vespa.athenz.tls.KeyUtils; +import org.junit.Test; + +import java.net.URI; +import java.security.KeyPair; +import java.time.Instant; +import java.util.Arrays; +import java.util.HashSet; + +import static com.yahoo.vespa.athenz.identityprovider.api.IdentityType.TENANT; +import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION; +import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.DEFAULT_KEY_VERSION; +import static org.junit.Assert.*; + +/** + * @author bjorncs + */ +public class IdentityDocumentSignerTest { + + @Test + public void generates_and_validates_signature() { + IdentityDocumentSigner signer = new IdentityDocumentSigner(); + IdentityType identityType = TENANT; + VespaUniqueInstanceId id = + new VespaUniqueInstanceId(1, "cluster-id", "instance", "application", "tenant", "region", "environment", identityType); + AthenzService providerService = new AthenzService("vespa", "service"); + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); + String configserverHostname = "configserverhostname"; + String instanceHostname = "instancehostname"; + Instant createdAt = Instant.EPOCH; + HashSet<String> ipAddresses = new HashSet<>(Arrays.asList("1.2.3.4", "::1")); + String signature = + signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, ipAddresses, identityType, keyPair.getPrivate()); + + SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( + null, signature, DEFAULT_KEY_VERSION, id, "dns-suffix", providerService, URI.create("https://zts"), + DEFAULT_DOCUMENT_VERSION, configserverHostname, instanceHostname, createdAt, ipAddresses, identityType); + + assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); + } + +}
\ No newline at end of file diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/TestUtils.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/TestUtils.java index 64f15408313..2a9b54f9e9e 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/TestUtils.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/TestUtils.java @@ -24,7 +24,10 @@ class TestUtils { } static X509Certificate createCertificate(KeyPair keyPair) { - X500Principal subject = new X500Principal("CN=mysubject"); + return createCertificate(keyPair, new X500Principal("CN=mysubject")); + } + + static X509Certificate createCertificate(KeyPair keyPair, X500Principal subject) { return X509CertificateBuilder .fromKeypair( keyPair, subject, Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA256_WITH_RSA, 1) diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/X509CertificateUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/X509CertificateUtilsTest.java index 718c0e88972..4039bf36a5f 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/X509CertificateUtilsTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/X509CertificateUtilsTest.java @@ -7,6 +7,7 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Arrays; import java.util.List; import static com.yahoo.vespa.athenz.tls.SubjectAlternativeName.Type.DNS_NAME; @@ -24,15 +25,7 @@ public class X509CertificateUtilsTest { public void can_deserialize_serialized_pem_certificate() { KeyPair keypair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); X500Principal subject = new X500Principal("CN=myservice"); - X509Certificate cert = X509CertificateBuilder - .fromKeypair( - keypair, - subject, - Instant.now(), - Instant.now().plus(1, ChronoUnit.DAYS), - SignatureAlgorithm.SHA256_WITH_RSA, - 1) - .build(); + X509Certificate cert = TestUtils.createCertificate(keypair, subject); assertEquals(subject, cert.getSubjectX500Principal()); String pem = X509CertificateUtils.toPem(cert); assertThat(pem, containsString("BEGIN CERTIFICATE")); @@ -41,6 +34,20 @@ public class X509CertificateUtilsTest { assertEquals(subject, deserializedCert.getSubjectX500Principal()); } + @Test + public void can_deserialize_serialized_pem_certificate_list() { + KeyPair keypair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); + X500Principal subject1 = new X500Principal("CN=myservice"); + X509Certificate cert1 = TestUtils.createCertificate(keypair, subject1); + X500Principal subject2 = new X500Principal("CN=myservice"); + X509Certificate cert2 = TestUtils.createCertificate(keypair, subject2); + List<X509Certificate> certificateList = Arrays.asList(cert1, cert2); + String pem = X509CertificateUtils.toPem(certificateList); + List<X509Certificate> deserializedCertificateList = X509CertificateUtils.certificateListFromPem(pem); + assertEquals(2, certificateList.size()); + assertEquals(subject1, deserializedCertificateList.get(0).getSubjectX500Principal()); + assertEquals(subject2, deserializedCertificateList.get(1).getSubjectX500Principal()); + } @Test public void can_list_subject_alternative_names() { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index d48a86eea61..c2e94e14486 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core.communication; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.component.Vtag; import com.yahoo.vespa.http.client.config.ConnectionParams; import com.yahoo.vespa.http.client.config.Endpoint; @@ -36,6 +38,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -48,6 +51,7 @@ import java.util.zip.GZIPOutputStream; class ApacheGatewayConnection implements GatewayConnection { private static Logger log = Logger.getLogger(ApacheGatewayConnection.class.getName()); + private static final ObjectMapper mapper = new ObjectMapper(); private static final String PATH = "/reserved-for-internal-use/feedapi?"; private final List<Integer> SUPPORTED_VERSIONS = new ArrayList<>(); private static final byte[] START_OF_FEED_XML = "<vespafeed>\n".getBytes(StandardCharsets.UTF_8); @@ -253,7 +257,7 @@ class ApacheGatewayConnection implements GatewayConnection { throw e; } try { - verifyServerResponseCode(response.getStatusLine()); + verifyServerResponseCode(response); verifyServerVersion(response.getFirstHeader(Headers.VERSION)); verifySessionHeader(response.getFirstHeader(Headers.SESSION_ID)); } catch (ServerResponseException e) { @@ -263,7 +267,8 @@ class ApacheGatewayConnection implements GatewayConnection { return response.getEntity().getContent(); } - private void verifyServerResponseCode(StatusLine statusLine) throws ServerResponseException { + private void verifyServerResponseCode(HttpResponse response) throws ServerResponseException { + StatusLine statusLine = response.getStatusLine(); // We use code 261-299 to report errors related to internal transitive errors that the tenants should not care // about to avoid masking more serious errors. int statusCode = statusLine.getStatusCode(); @@ -273,7 +278,22 @@ class ApacheGatewayConnection implements GatewayConnection { if (statusCode == 299) { throw new ServerResponseException(429, "Too many requests."); } - throw new ServerResponseException(statusLine.getStatusCode(), statusLine.getReasonPhrase()); + String message = tryGetDetailedErrorMessage(response) + .orElseGet(statusLine::getReasonPhrase); + throw new ServerResponseException(statusLine.getStatusCode(), message); + } + + private static Optional<String> tryGetDetailedErrorMessage(HttpResponse response) { + Header contentType = response.getEntity().getContentType(); + if (contentType == null || !contentType.getValue().equalsIgnoreCase("application/json")) return Optional.empty(); + try (InputStream in = response.getEntity().getContent()) { + JsonNode jsonNode = mapper.readTree(in); + JsonNode message = jsonNode.get("message"); + if (message == null || message.textValue() == null) return Optional.empty(); + return Optional.of(response.getStatusLine().getReasonPhrase() + " - " + message.textValue()); + } catch (IOException e) { + return Optional.empty(); + } } private void verifySessionHeader(Header serverHeader) throws ServerResponseException { diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java index 3136526fd42..1e2d37884ef 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java @@ -17,7 +17,10 @@ import org.apache.http.StatusLine; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.InputStreamEntity; +import org.apache.http.message.BasicHeader; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.mockito.stubbing.Answer; import java.io.ByteArrayInputStream; @@ -25,16 +28,14 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.stub; @@ -45,6 +46,9 @@ import static org.mockito.Mockito.when; public class ApacheGatewayConnectionTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Test public void testProtocolV3() throws Exception { final Endpoint endpoint = Endpoint.create("hostname", 666, false); @@ -54,7 +58,6 @@ public class ApacheGatewayConnectionTest { .setEnableV3Protocol(true) .build(); final List<Document> documents = new ArrayList<>(); - final CountDownLatch verifyContentSentLatch = new CountDownLatch(1); final String vespaDocContent ="Hello, I a JSON doc."; final String docId = "42"; @@ -63,7 +66,6 @@ public class ApacheGatewayConnectionTest { // This is the fake server, takes header client ID and uses this as session Id. ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { final Header clientIdHeader = post.getFirstHeader(Headers.CLIENT_ID); - verifyContentSentLatch.countDown(); return httpResponse(clientIdHeader.getValue(), "3"); }); @@ -80,7 +82,6 @@ public class ApacheGatewayConnectionTest { documents.add(createDoc(docId, vespaDocContent, true)); apacheGatewayConnection.writeOperations(documents); - assertTrue(verifyContentSentLatch.await(10, TimeUnit.SECONDS)); } @Test(expected=IllegalArgumentException.class) @@ -139,8 +140,6 @@ public class ApacheGatewayConnectionTest { .build(); final List<Document> documents = new ArrayList<>(); - final CountDownLatch verifyContentSentLatch = new CountDownLatch(1); - final String vespaDocContent ="Hello, I a JSON doc."; final String docId = "42"; @@ -157,7 +156,6 @@ public class ApacheGatewayConnectionTest { assertNotNull(header); assertThat(header.getValue(), is(FeedParams.DataFormat.JSON_UTF8.name())); // Test is done. - verifyContentSentLatch.countDown(); return httpResponse("clientId", "3"); }); @@ -175,7 +173,6 @@ public class ApacheGatewayConnectionTest { documents.add(createDoc(docId, vespaDocContent, true)); apacheGatewayConnection.writeOperations(documents); - assertTrue(verifyContentSentLatch.await(10, TimeUnit.SECONDS)); } @Test @@ -202,8 +199,6 @@ public class ApacheGatewayConnectionTest { .build(); final List<Document> documents = new ArrayList<>(); - final CountDownLatch verifyContentSentLatch = new CountDownLatch(1); - final String vespaDocContent ="Hello, I am the document data."; final String docId = "42"; @@ -221,7 +216,6 @@ public class ApacheGatewayConnectionTest { assertThat(rawContent, is( doc.getOperationId() + " 38\n" + vespaHeaderText + vespaDocContent + "\n" + vespaFooterText)); - verifyContentSentLatch.countDown(); } return httpResponse("clientId", "3"); @@ -244,7 +238,6 @@ public class ApacheGatewayConnectionTest { documents.add(doc); apacheGatewayConnection.writeOperations(documents); - assertTrue(verifyContentSentLatch.await(10, TimeUnit.SECONDS)); } @Test @@ -259,15 +252,12 @@ public class ApacheGatewayConnectionTest { .addDynamicHeader("foo", headerProvider) .build(); - CountDownLatch verifyContentSentLatch = new CountDownLatch(1); - AtomicInteger counter = new AtomicInteger(1); ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { Header[] fooHeader = post.getHeaders("foo"); assertEquals(1, fooHeader.length); assertEquals("foo", fooHeader[0].getName()); assertEquals("v" + counter.getAndIncrement(), fooHeader[0].getValue()); - verifyContentSentLatch.countDown(); return httpResponse("clientId", "3"); }); @@ -287,11 +277,33 @@ public class ApacheGatewayConnectionTest { documents.add(createDoc("42", "content", true)); apacheGatewayConnection.writeOperations(documents); apacheGatewayConnection.writeOperations(documents); - assertTrue(verifyContentSentLatch.await(10, TimeUnit.SECONDS)); verify(headerProvider, times(3)).getHeaderValue(); // 1x connect(), 2x writeOperations() } + @Test + public void detailed_error_message_is_extracted_from_error_responses_with_json() throws IOException, ServerResponseException { + String reasonPhrase = "Unauthorized"; + String errorMessage = "Invalid credentials"; + expectedException.expect(ServerResponseException.class); + expectedException.expectMessage(reasonPhrase + " - " + errorMessage); + + ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> createErrorHttpResponse(401, reasonPhrase, errorMessage)); + + ApacheGatewayConnection apacheGatewayConnection = + new ApacheGatewayConnection( + Endpoint.create("hostname", 666, false), + new FeedParams.Builder().build(), + "", + new ConnectionParams.Builder().build(), + mockFactory, + "clientId"); + apacheGatewayConnection.connect(); + apacheGatewayConnection.handshake(); + + apacheGatewayConnection.writeOperations(Collections.singletonList(createDoc("42", "content", true))); + } + private static ApacheGatewayConnection.HttpClientFactory mockHttpClientFactory(HttpExecuteMock httpExecuteMock) throws IOException { ApacheGatewayConnection.HttpClientFactory mockFactory = mock(ApacheGatewayConnection.HttpClientFactory.class); @@ -355,4 +367,20 @@ public class ApacheGatewayConnectionTest { when(httpEntityMock.getContent()).thenReturn(inputs); return httpResponseMock; } + + private static HttpResponse createErrorHttpResponse(int statusCode, String reasonPhrase, String message) throws IOException { + HttpResponse response = mock(HttpResponse.class); + + StatusLine statusLine = mock(StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(statusCode); + when(statusLine.getReasonPhrase()).thenReturn(reasonPhrase); + when(response.getStatusLine()).thenReturn(statusLine); + + HttpEntity httpEntity = mock(HttpEntity.class); + when(httpEntity.getContentType()).thenReturn(new BasicHeader("Content-Type", "application/json")); + String json = String.format("{\"message\": \"%s\"}", message); + when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream(json.getBytes())); + when(response.getEntity()).thenReturn(httpEntity); + return response; + } } diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index 8061c497454..081d7df18a4 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -112,6 +112,7 @@ fixdir ${VESPA_USER} wheel 755 var/db/vespa/index fixdir ${VESPA_USER} wheel 755 var/db/vespa/logcontrol fixdir ${VESPA_USER} wheel 755 var/db/vespa/search fixdir ${VESPA_USER} wheel 755 var/jdisc_core +fixdir ${VESPA_USER} wheel 755 var/vespa fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache/configserver fixdir ${VESPA_USER} wheel 755 var/vespa/cache/config/ diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java index e14144508c1..c6b5a6cb4fe 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java @@ -11,7 +11,6 @@ import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; -@Deprecated public class StatusResponse extends HttpResponse { MetricManager manager; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java index cced94d2d85..b1a7b6dbdeb 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java @@ -9,7 +9,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -@Deprecated public class VespaFeedHandlerCompatibility extends ThreadedHttpRequestHandler { private final VespaFeedHandlerGet getHandler; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java index b2108b74543..70631e0e66c 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java @@ -11,7 +11,6 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.search.handler.SearchHandler; -@Deprecated public class VespaFeedHandlerGet extends ThreadedHttpRequestHandler { private final SearchHandler searchHandler; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java index ba78a9f4a0c..36ab8090e95 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java @@ -21,7 +21,6 @@ import java.io.BufferedReader; import java.io.InputStreamReader; import java.util.concurrent.Executor; -@Deprecated public class VespaFeedHandlerRemove extends VespaFeedHandlerBase { @Inject diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java index 72722a85ddd..04ca6798b4c 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java @@ -20,7 +20,6 @@ import com.yahoo.vespaclient.config.FeederConfig; import java.util.concurrent.Executor; -@Deprecated public class VespaFeedHandlerRemoveLocation extends VespaFeedHandlerBase { @Inject diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java index a209c1d6213..94ad18fbb51 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java @@ -15,7 +15,6 @@ import com.yahoo.metrics.MetricManager; import com.yahoo.metrics.MetricSet; import com.yahoo.vespaclient.config.FeederConfig; -@Deprecated public class VespaFeedHandlerStatus extends ThreadedHttpRequestHandler { private MetricManager manager; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java index bb5e8c056db..c9af0933799 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java @@ -14,7 +14,6 @@ import com.yahoo.search.handler.SearchHandler; /** * @author thomasg */ -@Deprecated public class VespaFeedHandlerVisit extends ThreadedHttpRequestHandler { private final SearchHandler searchHandler; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java index 08cc9c914eb..d1ed02209b2 100755 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java @@ -39,7 +39,6 @@ import java.util.zip.GZIPOutputStream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -@SuppressWarnings("deprecation") // Tests a deprecated class public class VespaFeedHandlerTestCase { private VespaFeedHandler feedHandler; diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml index dbb48c93f39..a7ac908279b 100644 --- a/vespaclient-java/pom.xml +++ b/vespaclient-java/pom.xml @@ -84,18 +84,27 @@ </configuration> </plugin> <plugin> - <artifactId>maven-assembly-plugin</artifactId> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> <configuration> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> + <finalName>${project.artifactId}-jar-with-dependencies</finalName> + <filters> + <filter> + <!-- Don't include signature files in uber jar (most likely from bouncycastle). --> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> </configuration> <executions> <execution> - <id>make-assembly</id> <phase>package</phase> <goals> - <goal>single</goal> + <goal>shade</goal> </goals> </execution> </executions> diff --git a/vespajlib/src/main/java/com/yahoo/lang/SettableOptional.java b/vespajlib/src/main/java/com/yahoo/lang/SettableOptional.java index db5dd76c124..e83a6c3f308 100644 --- a/vespajlib/src/main/java/com/yahoo/lang/SettableOptional.java +++ b/vespajlib/src/main/java/com/yahoo/lang/SettableOptional.java @@ -19,6 +19,11 @@ public final class SettableOptional<T> { /** Creates a new settable optional with the given value */ public SettableOptional(T value) { this.value = value; } + /** Creates a new settable optional with the given value, or an empty */ + public SettableOptional(Optional<T> value) { + this.value = value.isPresent() ? value.get() : null; + } + public boolean isPresent() { return value != null; } diff --git a/vespalib/src/vespa/vespalib/component/.gitignore b/vespalib/src/vespa/vespalib/component/.gitignore index 75fec0dcbdd..583460ae288 100644 --- a/vespalib/src/vespa/vespalib/component/.gitignore +++ b/vespalib/src/vespa/vespalib/component/.gitignore @@ -1,4 +1,3 @@ *.So .depend Makefile -getversion diff --git a/vespalog/src/main/java/com/yahoo/log/LogLevel.java b/vespalog/src/main/java/com/yahoo/log/LogLevel.java index e20dcbf8b30..2d63e6babe2 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogLevel.java +++ b/vespalog/src/main/java/com/yahoo/log/LogLevel.java @@ -95,7 +95,6 @@ public class LogLevel extends Level { javaToVespa.put(Level.FINEST, SPAM); // need the VESPA ones too - javaToVespa.put(UNKNOWN, UNKNOWN); javaToVespa.put(FATAL, FATAL); javaToVespa.put(ERROR, ERROR); javaToVespa.put(EVENT, EVENT); @@ -144,16 +143,27 @@ public class LogLevel extends Level { * @param level The Java loglevel we want mapped to its VESPA * counterpart * @return The VESPA LogLevel instance representing the corresponding - * log level or the UNKNOWN instance if the log level was - * unknown (ie. not contained in the mapping. Should never - * happen). + * log level (or nearest normal level numerically if not in map) */ public static Level getVespaLogLevel(Level level) { Level ll = javaToVespa.get(level); if (ll != null) { return ll; } - return UNKNOWN; + int lv = level.intValue(); + if (lv > WARNING.intValue()) { + return ERROR; + } + if (lv > INFO.intValue()) { + return WARNING; + } + if (lv > DEBUG.intValue()) { + return INFO; + } + if (lv > FINEST.intValue()) { + return DEBUG; + } + return SPAM; } /** diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 1ab5558e2b9..d71660a990f 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -42,8 +42,9 @@ public class Lock implements Mutex { throw new RuntimeException("Exception acquiring lock '" + lockPath + "'", e); } - if (! acquired) throw new UncheckedTimeoutException("Timed out after waiting " + timeout.toString() + - " to acquire lock '" + lockPath + "'"); + if (! acquired) + throw new UncheckedTimeoutException("Timed out after waiting " + timeout + + " to acquire lock '" + lockPath + "'"); } @Override |