diff options
310 files changed, 3279 insertions, 3822 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java deleted file mode 100644 index 0b64f206267..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.Zone; -import com.yahoo.jdisc.http.ssl.SslKeyStoreConfigurator; -import com.yahoo.jdisc.http.ssl.SslKeyStoreContext; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.athenz.client.zts.Identity; -import com.yahoo.vespa.athenz.client.zts.ZtsClient; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; -import com.yahoo.security.KeyUtils; -import com.yahoo.vespa.athenz.utils.SiaUtils; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; - -import java.net.URI; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.KeyPair; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.PrivateKey; -import java.security.PublicKey; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils.getZoneConfig; - -/** - * A component that is responsible for retrieving an Athenz TLS certificate and configuring the configserver to use - * that certificate for its HTTPS endpoint. - * - * @author bjorncs - */ -@SuppressWarnings("unused") // Component injected into Jetty connector factory -public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements SslKeyStoreConfigurator { - private static final Logger log = Logger.getLogger(AthenzSslKeyStoreConfigurator.class.getName()); - private static final String CERTIFICATE_ALIAS = "athenz"; - private static final Duration EXPIRATION_MARGIN = Duration.ofHours(6); - private static final Path VESPA_SIA_DIRECTORY = Paths.get(Defaults.getDefaults().underVespaHome("var/vespa/sia")); - - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); - private final ZtsClient ztsClient; - private final KeyProvider keyProvider; - private final AthenzProviderServiceConfig.Zones zoneConfig; - private final Duration updatePeriod; - private final AthenzService configserverIdentity; - private volatile KeyStoreAndPassword currentKeyStore; - - @Inject - public AthenzSslKeyStoreConfigurator(ServiceIdentityProvider bootstrapIdentity, - KeyProvider keyProvider, - AthenzProviderServiceConfig config, - Zone zone, - ConfigserverConfig configserverConfig) { - AthenzProviderServiceConfig.Zones zoneConfig = getZoneConfig(config, zone); - AthenzService configserverIdentity = new AthenzService(zoneConfig.domain(), zoneConfig.serviceName()); - Duration updatePeriod = Duration.ofDays(config.updatePeriodDays()); - DefaultZtsClient ztsClient = new DefaultZtsClient(URI.create(zoneConfig.ztsUrl()), bootstrapIdentity); - this.ztsClient = ztsClient; - this.keyProvider = keyProvider; - this.zoneConfig = zoneConfig; - this.currentKeyStore = initializeKeystore(configserverIdentity, keyProvider, ztsClient, zoneConfig, updatePeriod); - this.updatePeriod = updatePeriod; - this.configserverIdentity = configserverIdentity; - } - - private static KeyStoreAndPassword initializeKeystore(AthenzService configserverIdentity, - KeyProvider keyProvider, - ZtsClient ztsClient, - AthenzProviderServiceConfig.Zones keystoreCacheDirectory, - Duration updatePeriod) { - return tryReadKeystoreFile(configserverIdentity, updatePeriod) - .orElseGet(() -> downloadCertificate(configserverIdentity, keyProvider, ztsClient, keystoreCacheDirectory)); - } - - private static Optional<KeyStoreAndPassword> tryReadKeystoreFile(AthenzService configserverIdentity, - Duration updatePeriod) { - Optional<X509Certificate> certificate = SiaUtils.readCertificateFile(VESPA_SIA_DIRECTORY, configserverIdentity); - if (!certificate.isPresent()) return Optional.empty(); - Optional<PrivateKey> privateKey = SiaUtils.readPrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity); - if (!privateKey.isPresent()) return Optional.empty(); - Instant minimumExpiration = Instant.now().plus(updatePeriod).plus(EXPIRATION_MARGIN); - boolean isExpired = certificate.get().getNotAfter().toInstant().isBefore(minimumExpiration); - if (isExpired) return Optional.empty(); - char[] password = generateKeystorePassword(); - KeyStore keyStore = KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry(CERTIFICATE_ALIAS, privateKey.get(), password, certificate.get()) - .build(); - return Optional.of(new KeyStoreAndPassword(keyStore, password)); - } - - @Override - public void configure(SslKeyStoreContext sslKeyStoreContext) { - sslKeyStoreContext.updateKeyStore(currentKeyStore.keyStore, new String(currentKeyStore.password)); - scheduler.scheduleAtFixedRate(new AthenzCertificateUpdater(sslKeyStoreContext), - updatePeriod.toDays()/*initial delay*/, - updatePeriod.toDays(), - TimeUnit.DAYS); - } - - @Override - public void deconstruct() { - try { - scheduler.shutdownNow(); - scheduler.awaitTermination(30, TimeUnit.SECONDS); - ztsClient.close(); - } catch (InterruptedException e) { - throw new RuntimeException("Failed to shutdown Athenz certificate updater on time", e); - } - } - - Instant getCertificateExpiry() throws KeyStoreException { - return getCertificateExpiry(currentKeyStore.keyStore); - } - - private static Instant getCertificateExpiry(KeyStore keyStore) throws KeyStoreException { - X509Certificate certificate = (X509Certificate) keyStore.getCertificate(CERTIFICATE_ALIAS); - return certificate.getNotAfter().toInstant(); - } - - private static KeyStoreAndPassword downloadCertificate(AthenzService configserverIdentity, - KeyProvider keyProvider, - ZtsClient ztsClient, - AthenzProviderServiceConfig.Zones zoneConfig) { - PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); - PublicKey publicKey = KeyUtils.extractPublicKey(privateKey); - Identity serviceIdentity = ztsClient.getServiceIdentity(configserverIdentity, - Integer.toString(zoneConfig.secretVersion()), - new KeyPair(publicKey, privateKey), - zoneConfig.certDnsSuffix()); - X509Certificate certificate = serviceIdentity.certificate(); - writeCredentials(configserverIdentity, certificate, serviceIdentity.caCertificates(), privateKey); - Instant expirationTime = certificate.getNotAfter().toInstant(); - Duration expiry = Duration.between(certificate.getNotBefore().toInstant(), expirationTime); - log.log(LogLevel.INFO, String.format("Got Athenz x509 certificate with expiry %s (expires %s)", expiry, expirationTime)); - - char[] keystorePassword = generateKeystorePassword(); - KeyStore keyStore = KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry(CERTIFICATE_ALIAS, privateKey, keystorePassword, certificate) - .build(); - return new KeyStoreAndPassword(keyStore, keystorePassword); - } - - private static void writeCredentials(AthenzService configserverIdentity, - X509Certificate certificate, - List<X509Certificate> caCertificates, - PrivateKey privateKey) { - SiaUtils.writeCertificateFile(VESPA_SIA_DIRECTORY, configserverIdentity, certificate); - SiaUtils.writePrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity, privateKey); - } - - private static char[] generateKeystorePassword() { - return UUID.randomUUID().toString().toCharArray(); - } - - private class AthenzCertificateUpdater implements Runnable { - - private final SslKeyStoreContext sslKeyStoreContext; - - AthenzCertificateUpdater(SslKeyStoreContext sslKeyStoreContext) { - this.sslKeyStoreContext = sslKeyStoreContext; - } - - @Override - public void run() { - try { - log.log(LogLevel.INFO, "Updating Athenz certificate from ZTS"); - currentKeyStore = downloadCertificate(configserverIdentity, keyProvider, ztsClient, zoneConfig); - sslKeyStoreContext.updateKeyStore(currentKeyStore.keyStore, new String(currentKeyStore.password)); - log.log(LogLevel.INFO, "Athenz certificate reload successfully completed"); - } catch (Throwable e) { - log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + e.getMessage(), e); - } - } - - } - - private static class KeyStoreAndPassword { - final KeyStore keyStore; - final char[] password; - - KeyStoreAndPassword(KeyStore keyStore, char[] password) { - this.keyStore = keyStore; - this.password = password; - } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java deleted file mode 100644 index da5fd430f1c..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.google.inject.Inject; -import com.yahoo.jdisc.http.ssl.SslTrustStoreConfigurator; -import com.yahoo.jdisc.http.ssl.SslTrustStoreContext; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; - -import java.nio.file.Paths; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.cert.X509Certificate; -import java.time.Instant; - -/** - * Programmatic configuration of configserver's truststore - * - * @author bjorncs - */ -public class AthenzSslTrustStoreConfigurator implements SslTrustStoreConfigurator { - - private static final String CERTIFICATE_ALIAS = "cfgselfsigned"; - - private final KeyStore trustStore; - - @Inject - public AthenzSslTrustStoreConfigurator(AthenzProviderServiceConfig athenzProviderServiceConfig) { - this.trustStore = createTrustStore(athenzProviderServiceConfig); - } - - @Override - public void configure(SslTrustStoreContext sslTrustStoreContext) { - sslTrustStoreContext.updateTrustStore(trustStore); - } - - Instant getTrustStoreExpiry() throws KeyStoreException { - X509Certificate certificate = (X509Certificate) trustStore.getCertificate(CERTIFICATE_ALIAS); - return certificate.getNotAfter().toInstant(); - } - - private static KeyStore createTrustStore(AthenzProviderServiceConfig athenzProviderServiceConfig) { - try { - return KeyStoreBuilder.withType(KeyStoreType.JKS) - .fromFile(Paths.get(athenzProviderServiceConfig.athenzCaTrustStore())) - .build(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java index 2d80b15c7ec..cd69099ea80 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java @@ -1,12 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.athenz.instanceproviderservice; +import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; -import com.google.inject.Inject; - -import java.security.KeyStoreException; import java.time.Duration; import java.time.Instant; import java.util.concurrent.Executors; @@ -21,23 +19,18 @@ import java.util.logging.Logger; public class CertificateExpiryMetricUpdater extends AbstractComponent { private static final Duration METRIC_REFRESH_PERIOD = Duration.ofMinutes(5); - private static final String NODE_CA_CERT_METRIC_NAME = "node-ca-cert.expiry.seconds"; private static final String ATHENZ_CONFIGSERVER_CERT_METRIC_NAME = "athenz-configserver-cert.expiry.seconds"; private final Logger logger = Logger.getLogger(CertificateExpiryMetricUpdater.class.getName()); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); private final Metric metric; - private final AthenzSslKeyStoreConfigurator keyStoreConfigurator; - private final AthenzSslTrustStoreConfigurator trustStoreConfigurator; + private final ConfigserverSslContextFactoryProvider provider; @Inject public CertificateExpiryMetricUpdater(Metric metric, - AthenzSslKeyStoreConfigurator keyStoreConfigurator, - AthenzSslTrustStoreConfigurator trustStoreConfigurator) { + ConfigserverSslContextFactoryProvider provider) { this.metric = metric; - this.keyStoreConfigurator = keyStoreConfigurator; - this.trustStoreConfigurator = trustStoreConfigurator; - + this.provider = provider; scheduler.scheduleAtFixedRate(this::updateMetrics, 30/*initial delay*/, @@ -56,20 +49,11 @@ public class CertificateExpiryMetricUpdater extends AbstractComponent { } private void updateMetrics() { - Instant now = Instant.now(); - try { - Duration keyStoreExpiry = Duration.between(now, keyStoreConfigurator.getCertificateExpiry()); + Duration keyStoreExpiry = Duration.between(Instant.now(), provider.getCertificateNotAfter()); metric.set(ATHENZ_CONFIGSERVER_CERT_METRIC_NAME, keyStoreExpiry.getSeconds(), null); - } catch (KeyStoreException e) { - logger.log(Level.WARNING, "Failed to update key store expiry metric", e); - } - - try { - Duration trustStoreExpiry = Duration.between(now, trustStoreConfigurator.getTrustStoreExpiry()); - metric.set(NODE_CA_CERT_METRIC_NAME, trustStoreExpiry.getSeconds(), null); - } catch (KeyStoreException e) { - logger.log(Level.WARNING, "Failed to update trust store expiry metric", e); + } catch (Exception e) { + logger.log(Level.WARNING, "Failed to update key store expiry metric: " + e.getMessage(), e); } } } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java new file mode 100644 index 00000000000..94df93aaea7 --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -0,0 +1,200 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.Zone; +import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.log.LogLevel; +import com.yahoo.security.KeyStoreBuilder; +import com.yahoo.security.KeyStoreType; +import com.yahoo.security.KeyUtils; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; +import com.yahoo.vespa.athenz.client.zts.Identity; +import com.yahoo.vespa.athenz.client.zts.ZtsClient; +import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import com.yahoo.vespa.athenz.utils.SiaUtils; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.GeneralSecurityException; +import java.security.KeyPair; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils.getZoneConfig; + +/** + * Configures the JDisc https connector with the configserver's Athenz provider certificate and private key. + * + * @author bjorncs + */ +public class ConfigserverSslContextFactoryProvider extends AbstractComponent implements SslContextFactoryProvider { + private static final String CERTIFICATE_ALIAS = "athenz"; + private static final Duration EXPIRATION_MARGIN = Duration.ofHours(6); + private static final Path VESPA_SIA_DIRECTORY = Paths.get(Defaults.getDefaults().underVespaHome("var/vespa/sia")); + + private static final Logger log = Logger.getLogger(ConfigserverSslContextFactoryProvider.class.getName()); + + private final SslContextFactory sslContextFactory; + private final ScheduledExecutorService scheduler = + Executors.newSingleThreadScheduledExecutor(runnable -> new Thread(runnable, "configserver-ssl-context-factory-provider")); + private final ZtsClient ztsClient; + private final KeyProvider keyProvider; + private final AthenzProviderServiceConfig.Zones zoneConfig; + private final AthenzService configserverIdentity; + + @Inject + public ConfigserverSslContextFactoryProvider(ServiceIdentityProvider bootstrapIdentity, + KeyProvider keyProvider, + AthenzProviderServiceConfig config, + Zone zone) { + this.zoneConfig = getZoneConfig(config, zone); + this.ztsClient = new DefaultZtsClient(URI.create(zoneConfig.ztsUrl()), bootstrapIdentity); + this.keyProvider = keyProvider; + this.configserverIdentity = new AthenzService(zoneConfig.domain(), zoneConfig.serviceName()); + + Duration updatePeriod = Duration.ofDays(config.updatePeriodDays()); + Path trustStoreFile = Paths.get(config.athenzCaTrustStore()); + this.sslContextFactory = initializeSslContextFactory(keyProvider, trustStoreFile, updatePeriod, configserverIdentity, ztsClient, zoneConfig); + scheduler.scheduleAtFixedRate(new KeystoreUpdater(sslContextFactory), + updatePeriod.toDays()/*initial delay*/, + updatePeriod.toDays(), + TimeUnit.DAYS); + } + + @Override + public SslContextFactory getInstance(String containerId, int port) { + return sslContextFactory; + } + + Instant getCertificateNotAfter() { + try { + X509Certificate certificate = (X509Certificate) sslContextFactory.getKeyStore().getCertificate(CERTIFICATE_ALIAS); + return certificate.getNotAfter().toInstant(); + } catch (GeneralSecurityException e) { + throw new IllegalStateException("Unable to find configserver certificate from keystore: " + e.getMessage(), e); + } + } + + @Override + public void deconstruct() { + try { + scheduler.shutdownNow(); + scheduler.awaitTermination(30, TimeUnit.SECONDS); + ztsClient.close(); + } catch (InterruptedException e) { + throw new RuntimeException("Failed to shutdown Athenz certificate updater on time", e); + } + } + + private static SslContextFactory initializeSslContextFactory(KeyProvider keyProvider, + Path trustStoreFile, + Duration updatePeriod, + AthenzService configserverIdentity, + ZtsClient ztsClient, + AthenzProviderServiceConfig.Zones zoneConfig) { + SslContextFactory factory = new SslContextFactory(); + + // Allow safe TLS_RSA* ciphers + String[] excludedCiphersWithoutTlsRsaExclusion = Arrays.stream(factory.getExcludeCipherSuites()) + .filter(cipher -> !cipher.equals("^TLS_RSA_.*$")) + .toArray(String[]::new); + factory.setExcludeCipherSuites(excludedCiphersWithoutTlsRsaExclusion); + + factory.setWantClientAuth(true); + + KeyStore trustStore = + KeyStoreBuilder.withType(KeyStoreType.JKS) + .fromFile(trustStoreFile) + .build(); + factory.setTrustStore(trustStore); + + KeyStore keyStore = + tryReadKeystoreFile(configserverIdentity, updatePeriod) + .orElseGet(() -> updateKeystore(configserverIdentity, generateKeystorePassword(), keyProvider, ztsClient, zoneConfig)); + factory.setKeyStore(keyStore); + factory.setKeyStorePassword(""); + return factory; + } + + private static Optional<KeyStore> tryReadKeystoreFile(AthenzService configserverIdentity, Duration updatePeriod) { + Optional<X509Certificate> certificate = SiaUtils.readCertificateFile(VESPA_SIA_DIRECTORY, configserverIdentity); + if (!certificate.isPresent()) return Optional.empty(); + Optional<PrivateKey> privateKey = SiaUtils.readPrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity); + if (!privateKey.isPresent()) return Optional.empty(); + Instant minimumExpiration = Instant.now().plus(updatePeriod).plus(EXPIRATION_MARGIN); + boolean isExpired = certificate.get().getNotAfter().toInstant().isBefore(minimumExpiration); + if (isExpired) return Optional.empty(); + KeyStore keyStore = KeyStoreBuilder.withType(KeyStoreType.JKS) + .withKeyEntry(CERTIFICATE_ALIAS, privateKey.get(), certificate.get()) + .build(); + return Optional.of(keyStore); + } + + private static KeyStore updateKeystore(AthenzService configserverIdentity, + char[] keystorePwd, + KeyProvider keyProvider, + ZtsClient ztsClient, + AthenzProviderServiceConfig.Zones zoneConfig) { + PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); + PublicKey publicKey = KeyUtils.extractPublicKey(privateKey); + Identity serviceIdentity = ztsClient.getServiceIdentity(configserverIdentity, + Integer.toString(zoneConfig.secretVersion()), + new KeyPair(publicKey, privateKey), + zoneConfig.certDnsSuffix()); + X509Certificate certificate = serviceIdentity.certificate(); + SiaUtils.writeCertificateFile(VESPA_SIA_DIRECTORY, configserverIdentity, certificate); + SiaUtils.writePrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity, privateKey); + Instant expirationTime = certificate.getNotAfter().toInstant(); + Duration expiry = Duration.between(certificate.getNotBefore().toInstant(), expirationTime); + log.log(LogLevel.INFO, String.format("Got Athenz x509 certificate with expiry %s (expires %s)", expiry, expirationTime)); + return KeyStoreBuilder.withType(KeyStoreType.JKS) + .withKeyEntry(CERTIFICATE_ALIAS, privateKey, keystorePwd, certificate) + .build(); + } + + private static char[] generateKeystorePassword() { + return UUID.randomUUID().toString().toCharArray(); + } + + private class KeystoreUpdater implements Runnable { + final SslContextFactory sslContextFactory; + + KeystoreUpdater(SslContextFactory sslContextFactory) { + this.sslContextFactory = sslContextFactory; + } + + @Override + public void run() { + try { + log.log(LogLevel.INFO, "Updating configserver provider certificate from ZTS"); + char[] keystorePwd = generateKeystorePassword(); + KeyStore keyStore = updateKeystore(configserverIdentity, keystorePwd, keyProvider, ztsClient, zoneConfig); + sslContextFactory.reload(scf -> { + scf.setKeyStore(keyStore); + scf.setKeyStorePassword(new String(keystorePwd)); + }); + log.log(LogLevel.INFO, "Certificate successfully updated"); + } catch (Throwable t) { + log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + t.getMessage(), t); + } + } + } +} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java index ca6b5529b08..74e9b02e150 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java @@ -1,6 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.athenz.instanceproviderservice; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; + import java.security.KeyPair; import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; @@ -15,13 +18,7 @@ public class AutoGeneratedKeyProvider implements KeyProvider { private final KeyPair keyPair; public AutoGeneratedKeyProvider() { - try { - KeyPairGenerator rsa = KeyPairGenerator.getInstance("RSA"); - rsa.initialize(2048); - keyPair = rsa.genKeyPair(); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } + keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); } @Override diff --git a/bootstrap-cmake.sh b/bootstrap-cmake.sh index 0c2d9553213..b484d41e8b6 100755 --- a/bootstrap-cmake.sh +++ b/bootstrap-cmake.sh @@ -2,13 +2,28 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. usage() { - echo "Usage: $0 <source-dir> [<extra-cmake-args>]" >&2 + echo "Usage: $0 [-u] <source-dir> [<extra-cmake-args>]" >&2 } -if [[ $# -eq 1 && ( "$1" = "-h" || "$1" = "--help" )]]; then - usage - exit 0 -elif [[ $# -eq 1 ]]; then +UNPRIVILEGED=false +while getopts "uh" opt; do + case "${opt}" in + u) + UNPRIVILEGED=true + ;; + h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [[ $# -eq 1 ]]; then SOURCE_DIR=$1 EXTRA_CMAKE_ARGS="" elif [ $# -eq 2 ]; then @@ -24,12 +39,21 @@ if [ -z "$VESPA_LLVM_VERSION" ]; then VESPA_LLVM_VERSION=5.0 fi +if $UNPRIVILEGED; then + VESPA_INSTALL_PREFIX="$HOME/vespa" + UNPRIVILEGED_ARGS="-DVESPA_USER=$(id -un) -DVESPA_UNPRIVILEGED=yes" +else + VESPA_INSTALL_PREFIX="/opt/vespa" + UNPRIVILEGED_ARGS="" +fi + cmake3 \ - -DCMAKE_INSTALL_PREFIX=/opt/vespa \ + -DCMAKE_INSTALL_PREFIX=${VESPA_INSTALL_PREFIX} \ -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ -DEXTRA_LINK_DIRECTORY="/opt/vespa-gtest/lib;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \ -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-gtest/include;/opt/vespa-boost/include;/opt/vespa-cppunit/include;/usr/include/llvm$VESPA_LLVM_VERSION" \ - -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-gtest/lib;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \ + -DCMAKE_INSTALL_RPATH="${VESPA_INSTALL_PREFIX}/lib64;/opt/vespa-gtest/lib;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \ + ${UNPRIVILEGED_ARGS} \ ${EXTRA_CMAKE_ARGS} \ -DVESPA_LLVM_VERSION=$VESPA_LLVM_VERSION \ "${SOURCE_DIR}" diff --git a/bootstrap-cpp.sh b/bootstrap-cpp.sh index e6b4c816065..bc4c6466596 100755 --- a/bootstrap-cpp.sh +++ b/bootstrap-cpp.sh @@ -2,16 +2,31 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. usage() { - echo "Usage: $0 <source-dir> <build-dir>" >&2 + echo "Usage: $0 [-u] <source-dir> <build-dir>" >&2 } +UNPRIVILEGED="" +while getopts "uh" opt; do + case "${opt}" in + u) + UNPRIVILEGED="-u" + ;; + h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + # Parse arguments if [ $# -eq 2 ]; then SOURCE_DIR="$1" BUILD_DIR="$2" -elif [[ $# -eq 1 && ( "$1" = "-h" || "$1" = "--help" )]]; then - usage - exit 0 else echo "Wrong number of arguments: expected 2, was $#" >&2 usage @@ -37,4 +52,4 @@ source /opt/rh/devtoolset-7/enable || true cd "${SOURCE_DIR}" bash ./bootstrap.sh full cd "${BUILD_DIR}" -bash ${SOURCE_DIR}/bootstrap-cmake.sh "${SOURCE_DIR}" +bash ${SOURCE_DIR}/bootstrap-cmake.sh ${UNPRIVILEGED} "${SOURCE_DIR}" diff --git a/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java b/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java index ccee4844b49..38ca08ecff1 100644 --- a/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java +++ b/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java @@ -2,6 +2,7 @@ package com.yahoo; import com.yahoo.osgi.maven.ProjectBundleClassPaths; +import com.yahoo.vespa.config.VespaVersion; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -64,9 +65,12 @@ public class BundleIT { } @Test - @Ignore // TODO Vespa 7: Should we fix this? Why not? - public void require_that_bundle_version_matches_pom_version() { - assertThat(mainAttributes.getValue("Bundle-Version"), is("5.1.0")); + public void require_that_bundle_version_is_added_to_manifest() { + String bundleVersion = mainAttributes.getValue("Bundle-Version"); + + // Because of snapshot builds, we can only verify the major version. + int majorBundleVersion = Integer.valueOf(bundleVersion.substring(0, bundleVersion.indexOf('.'))); + assertThat(majorBundleVersion, is(VespaVersion.major)); } @Test diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index d2ffe10677b..8fff69e11a6 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -13,6 +13,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder; import com.yahoo.vespa.model.builder.xml.dom.DomAdminV4Builder; +import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import org.w3c.dom.Element; @@ -48,6 +49,7 @@ public class AdminModel extends ConfigModel { if (admin == null) return; if (admin.getClusterControllers() != null) admin.getClusterControllers().prepare(); + admin.getLogServerContainerCluster().ifPresent(ContainerCluster::prepare); } private void verifyClusterControllersOnlyDefinedForContent(ConfigModelRepo configModelRepo) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 757dab4cbf3..4442b42bb88 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -63,7 +63,7 @@ public class MockApplicationPackage implements ApplicationPackage { this.failOnValidateXml = failOnValidateXml; queryProfileRegistry = new QueryProfileXMLReader().read(asNamedReaderList(queryProfileType), asNamedReaderList(queryProfile)); - applicationMetaData = new ApplicationMetaData("user", "dir", 0L, false, "application", "checksum", 0L, 0L); + applicationMetaData = new ApplicationMetaData("user", "dir", 0L, false, "application", "checksum", 1L, 0L); } /** Returns the root of this application package relative to the current dir */ diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java new file mode 100644 index 00000000000..9335c0b4005 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java @@ -0,0 +1,35 @@ +package com.yahoo.searchdefinition; + +import java.util.List; + +/** + * A rank profile which ignores all calls made to it which may fail in a document only setting. + * This is used by the search definition parser when it is requested to parse documents only, + * to avoid having to check for this in every method which adds to the rank profile. + * (And why do we ever want to parse documents only? Because it is used when generating Java classes + * from documents, where the full application package may not be available.) + * + * @author bratseth + */ +public class DocumentsOnlyRankProfile extends RankProfile { + + public DocumentsOnlyRankProfile(String name, Search search, RankProfileRegistry rankProfileRegistry) { + super(name, search, rankProfileRegistry); + } + + @Override + public void setFirstPhaseRanking(String expression) { + // Ignore + } + + @Override + public void setSecondPhaseRanking(String expression) { + // Ignore + } + + @Override + public void addFunction(String name, List<String> arguments, String expression, boolean inline) { + // Ignore + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index afd33da369f..0d9ea00bf73 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Stack; import java.util.stream.Collectors; /** @@ -71,7 +70,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement currentResolutionCallStack.stream().map(Reference::toString).collect(Collectors.joining(" -> ")) + " -> " + reference); - // A reference to a macro argument? + // A reference to a function argument? Optional<String> binding = boundIdentifier(reference); if (binding.isPresent()) { try { @@ -117,7 +116,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement } /** - * Returns the default type for this simple feature, or nullif it does not have a default + * Returns the default type for this simple feature, or null if it does not have a default */ public TensorType defaultTypeOf(Reference reference) { if ( ! FeatureNames.isSimpleFeature(reference)) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index b7e1f9d4538..16e494c2db1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -95,13 +95,9 @@ public class RankProfile implements Serializable, Cloneable { /** The properties of this - a multimap */ private Map<String, List<RankProperty>> rankProperties = new LinkedHashMap<>(); - private Boolean ignoreDefaultRankFeatures=null; + private Boolean ignoreDefaultRankFeatures = null; - private String secondPhaseRankingString=null; - - private String firstPhaseRankingString=null; - - private Map<String, Macro> macros= new LinkedHashMap<>(); + private Map<String, RankingExpressionFunction> functions = new LinkedHashMap<>(); private Set<String> filterFields = new HashSet<>(); @@ -339,13 +335,22 @@ public class RankProfile implements Serializable, Cloneable { * Returns null if no expression is set. */ public RankingExpression getFirstPhaseRanking() { - if (firstPhaseRanking!=null) return firstPhaseRanking; - if (getInherited()!=null) return getInherited().getFirstPhaseRanking(); + if (firstPhaseRanking != null) return firstPhaseRanking; + if (getInherited() != null) return getInherited().getFirstPhaseRanking(); return null; } public void setFirstPhaseRanking(RankingExpression rankingExpression) { - this.firstPhaseRanking=rankingExpression; + this.firstPhaseRanking = rankingExpression; + } + + public void setFirstPhaseRanking(String expression) { + try { + this.firstPhaseRanking = parseRankingExpression("firstphase", expression); + } + catch (ParseException e) { + throw new IllegalArgumentException("Illegal first phase ranking function", e); + } } /** @@ -353,31 +358,22 @@ public class RankProfile implements Serializable, Cloneable { * Returns null if no expression is set. */ public RankingExpression getSecondPhaseRanking() { - if (secondPhaseRanking!=null) return secondPhaseRanking; - if (getInherited()!=null) return getInherited().getSecondPhaseRanking(); + if (secondPhaseRanking != null) return secondPhaseRanking; + if (getInherited() != null) return getInherited().getSecondPhaseRanking(); return null; } public void setSecondPhaseRanking(RankingExpression rankingExpression) { - this.secondPhaseRanking=rankingExpression; + this.secondPhaseRanking = rankingExpression; } - /** - * Called by parser to store the expression string, for delayed evaluation - * - * @param exp ranking expression for second phase - */ - public void setSecondPhaseRankingString(String exp) { - this.secondPhaseRankingString = exp; - } - - /** - * Called by parser to store the expression string, for delayed evaluation - * - * @param exp ranking expression for first phase - */ - public void setFirstPhaseRankingString(String exp) { - this.firstPhaseRankingString = exp; + public void setSecondPhaseRanking(String expression) { + try { + this.secondPhaseRanking = parseRankingExpression("secondphase", expression); + } + catch (ParseException e) { + throw new IllegalArgumentException("Illegal second phase ranking function", e); + } } /** Returns a read-only view of the summary features to use in this profile. This is never null */ @@ -412,8 +408,8 @@ public class RankProfile implements Serializable, Cloneable { } public void addRankFeature(ReferenceNode feature) { - if (rankFeatures==null) - rankFeatures=new LinkedHashSet<>(); + if (rankFeatures == null) + rankFeatures = new LinkedHashSet<>(); rankFeatures.add(feature); } @@ -522,55 +518,43 @@ public class RankProfile implements Serializable, Cloneable { } public boolean getIgnoreDefaultRankFeatures() { - if (ignoreDefaultRankFeatures!=null) return ignoreDefaultRankFeatures; - return (getInherited()!=null) && getInherited().getIgnoreDefaultRankFeatures(); + if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; + return (getInherited() != null) && getInherited().getIgnoreDefaultRankFeatures(); } - /** - * Returns the string form of the second phase ranking expression. - * - * @return string form of second phase ranking expression - */ - public String getSecondPhaseRankingString() { - if (secondPhaseRankingString != null) return secondPhaseRankingString; - if (getInherited() != null) return getInherited().getSecondPhaseRankingString(); - return null; - } - - /** - * Returns the string form of the first phase ranking expression. - * - * @return string form of first phase ranking expression - */ - public String getFirstPhaseRankingString() { - if (firstPhaseRankingString != null) return firstPhaseRankingString; - if (getInherited() != null) return getInherited().getFirstPhaseRankingString(); - return null; + /** Adds a function */ + public void addFunction(String name, List<String> arguments, String expression, boolean inline) { + try { + addFunction(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); + } + catch (ParseException e) { + throw new IllegalArgumentException("Could not parse function '" + name + "'", e); + } } - /** Creates a new (empty) macro and returns it */ - public Macro addMacro(String name, boolean inline) { - Macro macro = new Macro(name, inline); - macros.put(name, macro); - return macro; + /** Adds a function and returns it */ + public RankingExpressionFunction addFunction(ExpressionFunction function, boolean inline) { + RankingExpressionFunction rankingExpressionFunction = new RankingExpressionFunction(function, inline); + functions.put(function.getName(), rankingExpressionFunction); + return rankingExpressionFunction; } - /** Returns an unmodifiable view of the macros in this */ - public Map<String, Macro> getMacros() { - if (macros.size() == 0 && getInherited()==null) return Collections.emptyMap(); - if (macros.size() == 0) return getInherited().getMacros(); - if (getInherited() == null) return Collections.unmodifiableMap(macros); + /** Returns an unmodifiable view of the functions in this */ + public Map<String, RankingExpressionFunction> getFunctions() { + if (functions.size() == 0 && getInherited() == null) return Collections.emptyMap(); + if (functions.size() == 0) return getInherited().getFunctions(); + if (getInherited() == null) return Collections.unmodifiableMap(functions); // Neither is null - Map<String, Macro> allMacros = new LinkedHashMap<>(getInherited().getMacros()); - allMacros.putAll(macros); - return Collections.unmodifiableMap(allMacros); + Map<String, RankingExpressionFunction> allFunctions = new LinkedHashMap<>(getInherited().getFunctions()); + allFunctions.putAll(functions); + return Collections.unmodifiableMap(allFunctions); } public int getKeepRankCount() { - if (keepRankCount>=0) return keepRankCount; - if (getInherited()!=null) return getInherited().getKeepRankCount(); + if (keepRankCount >= 0) return keepRankCount; + if (getInherited() != null) return getInherited().getKeepRankCount(); return -1; } @@ -579,8 +563,8 @@ public class RankProfile implements Serializable, Cloneable { } public double getRankScoreDropLimit() { - if (rankScoreDropLimit>-Double.MAX_VALUE) return rankScoreDropLimit; - if (getInherited()!=null) return getInherited().getRankScoreDropLimit(); + if (rankScoreDropLimit >- Double.MAX_VALUE) return rankScoreDropLimit; + if (getInherited() != null) return getInherited().getRankScoreDropLimit(); return rankScoreDropLimit; } @@ -607,58 +591,15 @@ public class RankProfile implements Serializable, Cloneable { return retval; } - /** - * Will take the parser-set textual ranking expressions and turn into ranking expression objects, - * if not already done - */ - // TODO: There doesn't appear to be any good reason to defer parsing of ranking expressions - // until this is called. Simplify by parsing them right away. - public void parseExpressions() { - try { - parseRankingExpressions(); - parseMacros(); - } catch (ParseException e) { - throw new IllegalArgumentException(e); - } - } - - /** - * Passes the contents of macros on to parser. Then put all the implied rank properties - * from those macros into the profile's props map. - */ - private void parseMacros() throws ParseException { - for (Map.Entry<String, Macro> e : getMacros().entrySet()) { - String macroName = e.getKey(); - Macro macro = e.getValue(); - if (macro.getRankingExpression() == null) { - RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression()); - macro.setRankingExpression(expr); - macro.setTextualExpression(expr.getRoot().toString()); - } - } - } - - /** - * Passes ranking expressions on to parser - * - * @throws ParseException if either of the ranking expressions could not be parsed - */ - private void parseRankingExpressions() throws ParseException { - if (getFirstPhaseRankingString() != null && firstPhaseRanking == null) - setFirstPhaseRanking(parseRankingExpression("firstphase", getFirstPhaseRankingString())); - if (getSecondPhaseRankingString() != null && secondPhaseRanking == null) - setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString())); - } - - private RankingExpression parseRankingExpression(String expressionName, String exp) throws ParseException { - if (exp.trim().length() == 0) + private RankingExpression parseRankingExpression(String expressionName, String expression) throws ParseException { + if (expression.trim().length() == 0) throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expressionName + "."); - try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, exp.trim())) { + try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, expression.trim())) { return new RankingExpression(expressionName, rankingExpressionReader); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { - ParseException exception = new ParseException("Could not parse ranking expression '" + exp.trim() + + ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + "' in " + getName()+ ", " + expressionName + "."); throw (ParseException)exception.initCause(e); } @@ -686,14 +627,13 @@ public class RankProfile implements Serializable, Cloneable { @Override public RankProfile clone() { try { - // Note: This treats RankingExpression in Macros as immutables even though they are not RankProfile clone = (RankProfile)super.clone(); clone.rankSettings = new LinkedHashSet<>(this.rankSettings); clone.matchPhaseSettings = this.matchPhaseSettings; // hmm? clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null; clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null; clone.rankProperties = new LinkedHashMap<>(this.rankProperties); - clone.macros = new LinkedHashMap<>(this.macros); + clone.functions = new LinkedHashMap<>(this.functions); clone.filterFields = new HashSet<>(this.filterFields); clone.constants = new HashMap<>(this.constants); return clone; @@ -719,60 +659,59 @@ public class RankProfile implements Serializable, Cloneable { } private void compileThis(QueryProfileRegistry queryProfiles, ImportedModels importedModels) { - parseExpressions(); - checkNameCollisions(getMacros(), getConstants()); + checkNameCollisions(getFunctions(), getConstants()); ExpressionTransforms expressionTransforms = new ExpressionTransforms(); - // Macro compiling first pass: compile inline macros without resolving other macros - Map<String, Macro> inlineMacros = compileMacros(getInlineMacros(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms); + // Function compiling first pass: compile inline functions without resolving other functions + Map<String, RankingExpressionFunction> inlineFunctions = + compileFunctions(getInlineFunctions(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms); - // Macro compiling second pass: compile all macros and insert previously compiled inline macros - macros = compileMacros(getMacros(), queryProfiles, importedModels, inlineMacros, expressionTransforms); + // Function compiling second pass: compile all functions and insert previously compiled inline functions + functions = compileFunctions(getFunctions(), queryProfiles, importedModels, inlineFunctions, expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); } - private void checkNameCollisions(Map<String, Macro> macros, Map<String, Value> constants) { - for (Map.Entry<String, Macro> macroEntry : macros.entrySet()) { - if (constants.get(macroEntry.getKey()) != null) - throw new IllegalArgumentException("Cannot have both a constant and macro named '" + - macroEntry.getKey() + "'"); + private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { + for (Map.Entry<String, RankingExpressionFunction> functionEntry : functions.entrySet()) { + if (constants.get(functionEntry.getKey()) != null) + throw new IllegalArgumentException("Cannot have both a constant and function named '" + + functionEntry.getKey() + "'"); } } - private Map<String, Macro> getInlineMacros() { - return getMacros().entrySet().stream().filter(x -> x.getValue().getInline()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + private Map<String, RankingExpressionFunction> getInlineFunctions() { + return getFunctions().entrySet().stream().filter(x -> x.getValue().inline()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private Map<String, Macro> compileMacros(Map<String, Macro> macros, - QueryProfileRegistry queryProfiles, - ImportedModels importedModels, - Map<String, Macro> inlineMacros, - ExpressionTransforms expressionTransforms) { - Map<String, Macro> compiledMacros = new LinkedHashMap<>(); - for (Map.Entry<String, Macro> entry : macros.entrySet()) { - Macro macro = entry.getValue().clone(); - RankingExpression exp = compile(macro.getRankingExpression(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); - macro.setRankingExpression(exp); - compiledMacros.put(entry.getKey(), macro); + private Map<String, RankingExpressionFunction> compileFunctions(Map<String, RankingExpressionFunction> functions, + QueryProfileRegistry queryProfiles, + ImportedModels importedModels, + Map<String, RankingExpressionFunction> inlineFunctions, + ExpressionTransforms expressionTransforms) { + Map<String, RankingExpressionFunction> compiledFunctions = new LinkedHashMap<>(); + for (Map.Entry<String, RankingExpressionFunction> entry : functions.entrySet()) { + RankingExpressionFunction rankingExpressionFunction = entry.getValue(); + RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); + compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withBody(compiled)); } - return compiledMacros; + return compiledFunctions; } private RankingExpression compile(RankingExpression expression, QueryProfileRegistry queryProfiles, ImportedModels importedModels, Map<String, Value> constants, - Map<String, Macro> inlineMacros, + Map<String, RankingExpressionFunction> inlineFunctions, ExpressionTransforms expressionTransforms) { if (expression == null) return null; RankProfileTransformContext context = new RankProfileTransformContext(this, queryProfiles, importedModels, constants, - inlineMacros); + inlineFunctions); expression = expressionTransforms.transform(expression, context); for (Map.Entry<String, String> rankProperty : context.rankProperties().entrySet()) { addRankProperty(rankProperty.getKey(), rankProperty.getValue()); @@ -785,9 +724,9 @@ public class RankProfile implements Serializable, Cloneable { * referable from this rank profile. */ public TypeContext<Reference> typeContext(QueryProfileRegistry queryProfiles) { - MapEvaluationTypeContext context = new MapEvaluationTypeContext(getMacros().values().stream() - .map(Macro::asExpressionFunction) - .collect(Collectors.toList())); + MapEvaluationTypeContext context = new MapEvaluationTypeContext(getFunctions().values().stream() + .map(RankingExpressionFunction::function) + .collect(Collectors.toList())); // Add small and large constants, respectively getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type())); @@ -854,11 +793,11 @@ public class RankProfile implements Serializable, Cloneable { /** True if this setting really pertains to an index, not a field within an index */ private boolean isIndexLevel; - private Type(String name) { + Type(String name) { this(name,false); } - private Type(String name,boolean isIndexLevel) { + Type(String name,boolean isIndexLevel) { this.name = name; this.isIndexLevel=isIndexLevel; } @@ -866,7 +805,7 @@ public class RankProfile implements Serializable, Cloneable { /** True if this setting really pertains to an index, not a field within an index */ public boolean isIndexLevel() { return isIndexLevel; } - /** @return The name of this type */ + /** Returns the name of this type */ public String getName() { return name; } @@ -899,10 +838,12 @@ public class RankProfile implements Serializable, Cloneable { } } + @Override public int hashCode() { return fieldName.hashCode() + 17 * type.hashCode(); } + @Override public boolean equals(Object object) { if (!(object instanceof RankSetting)) { return false; @@ -913,6 +854,7 @@ public class RankProfile implements Serializable, Cloneable { type.equals(other.type); } + @Override public String toString() { return type + " setting " + fieldName + ": " + value; } @@ -936,7 +878,7 @@ public class RankProfile implements Serializable, Cloneable { @Override public int hashCode() { - return name.hashCode() + 17*value.hashCode(); + return name.hashCode() + 17 * value.hashCode(); } @Override @@ -953,73 +895,32 @@ public class RankProfile implements Serializable, Cloneable { } - /** - * Represents a declared macro in the profile. It is, after parsing, transformed into ExpressionMacro - */ - public static class Macro implements Serializable, Cloneable { + /** A function in a rank profile */ + public static class RankingExpressionFunction { - private final String name; - private String textualExpression = null; - private RankingExpression expression = null; - private List<String> formalParams = new ArrayList<>(); + private final ExpressionFunction function; - /** True if this should be inlined into calling expressions. Useful for very cheap macros. */ + /** True if this should be inlined into calling expressions. Useful for very cheap functions. */ private final boolean inline; - public Macro(String name, boolean inline) { - this.name = name; + public RankingExpressionFunction(ExpressionFunction function, boolean inline) { + this.function = function; this.inline = inline; } - public void addParam(String name) { - formalParams.add(name); - } - - public List<String> getFormalParams() { - return formalParams; - } - - public String getTextualExpression() { - return textualExpression; - } - - public void setTextualExpression(String textualExpression) { - this.textualExpression = textualExpression; - } - - public void setRankingExpression(RankingExpression expr) { - this.expression=expr; - } - - public RankingExpression getRankingExpression() { - return expression; - } - - public String getName() { - return name; - } + public ExpressionFunction function() { return function; } - public boolean getInline() { - return inline && formalParams.size() == 0; // only inline no-arg macros; + public boolean inline() { + return inline && function.arguments().isEmpty(); // only inline no-arg functions; } - public ExpressionFunction asExpressionFunction() { - return new ExpressionFunction(getName(), getFormalParams(), getRankingExpression()); - } - - @Override - public Macro clone() { - try { - return (Macro)super.clone(); - } - catch (CloneNotSupportedException e) { - throw new RuntimeException("Won't happen", e); - } + public RankingExpressionFunction withBody(RankingExpression expression) { + return new RankingExpressionFunction(function.withBody(expression), inline); } @Override public String toString() { - return "macro " + getName() + ": " + expression; + return "function " + function; } } @@ -1106,6 +1007,7 @@ public class RankProfile implements Serializable, Cloneable { public Map<String, String> getTypes() { return Collections.unmodifiableMap(types); } + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 91f86bb1c2a..3c2ebc058ac 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -39,17 +39,25 @@ public class SearchBuilder { private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); private List<Search> searchList = new LinkedList<>(); - private ApplicationPackage app = null; + private ApplicationPackage app; private boolean isBuilt = false; private DocumentModel model = new DocumentModel(); private final RankProfileRegistry rankProfileRegistry; private final QueryProfileRegistry queryProfileRegistry; + /** True to build the document aspect only, skipping instantiation of rank profiles */ + private final boolean documentsOnly; + /** For testing only */ public SearchBuilder() { this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry()); } + /** Used for generating documents for typed access to document fields in Java */ + public SearchBuilder(boolean documentsOnly) { + this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); + } + /** For testing only */ public SearchBuilder(ApplicationPackage app) { this(app, new RankProfileRegistry(), new QueryProfileRegistry()); @@ -68,9 +76,16 @@ public class SearchBuilder { public SearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { + this(app, rankProfileRegistry, queryProfileRegistry, false); + } + public SearchBuilder(ApplicationPackage app, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry, + boolean documentsOnly) { this.app = app; this.rankProfileRegistry = rankProfileRegistry; this.queryProfileRegistry = queryProfileRegistry; + this.documentsOnly = documentsOnly; } /** @@ -150,7 +165,7 @@ public class SearchBuilder { Search search; SimpleCharStream stream = new SimpleCharStream(str); try { - search = new SDParser(stream, deployLogger, app, rankProfileRegistry).search(docTypeMgr, searchDefDir); + search = new SDParser(stream, deployLogger, app, rankProfileRegistry, documentsOnly).search(docTypeMgr, searchDefDir); } catch (TokenMgrException e) { throw new ParseException("Unknown symbol: " + e.getMessage()); } catch (ParseException pe) { @@ -164,9 +179,9 @@ public class SearchBuilder { * {@link Search} object is considered to be "raw" if it has not already been processed. This is the case for most * programmatically constructed search objects used in unit tests. * - * @param rawSearch The object to import. - * @return The name of the imported object. - * @throws IllegalArgumentException Thrown if the given search object has already been processed. + * @param rawSearch the object to import. + * @return the name of the imported object. + * @throws IllegalArgumentException if the given search object has already been processed. */ public String importRawSearch(Search rawSearch) { if (rawSearch.getName() == null) { @@ -250,15 +265,15 @@ public class SearchBuilder { * #build()} method so that subclasses can choose not to build anything. */ protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles, boolean validate) { - Processing.process(search, deployLogger, rankProfileRegistry, queryProfiles, validate); + new Processing().process(search, deployLogger, rankProfileRegistry, queryProfiles, validate, documentsOnly); } /** * Convenience method to call {@link #getSearch(String)} when there is only a single {@link Search} object * built. This method will never return null. * - * @return The build object. - * @throws IllegalStateException Thrown if there is not exactly one search. + * @return the built object + * @throws IllegalStateException if there is not exactly one search. */ public Search getSearch() { if ( ! isBuilt) throw new IllegalStateException("Searches not built."); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java index c1b05c0fcdf..49b7ad621af 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java @@ -1,8 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.collections.Pair; + +import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.logging.Logger; @@ -54,22 +58,18 @@ public class FieldRankSettings { table.getType().equals(NativeTable.Type.REVERSE_PROXIMITY)); } - public Map<String,String> deriveRankProperties(int part) { - Map<String,String> ret = new LinkedHashMap<>(); - int i = part; - for (Iterator<NativeTable> itr = tables.values().iterator(); itr.hasNext(); ++i) { - NativeTable table = itr.next(); - if (isFieldMatchTable(table)) { - ret.put("nativeFieldMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); - } - if (isAttributeMatchTable(table)) { - ret.put("nativeAttributeMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); - } - if (isProximityTable(table)) { - ret.put("nativeProximity." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); - } + public List<Pair<String, String>> deriveRankProperties() { + List<Pair<String, String>> properties = new ArrayList<>(); + for (Iterator<NativeTable> i = tables.values().iterator(); i.hasNext();) { + NativeTable table = i.next(); + if (isFieldMatchTable(table)) + properties.add(new Pair<>("nativeFieldMatch." + table.getType().getName() + "." + fieldName, table.getName())); + if (isAttributeMatchTable(table)) + properties.add(new Pair<>("nativeAttributeMatch." + table.getType().getName() + "." + fieldName, table.getName())); + if (isProximityTable(table)) + properties.add(new Pair<>("nativeProximity." + table.getType().getName() + "." + fieldName, table.getName())); } - return ret; + return properties; } public String toString() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 43cc2fad285..c041d5c6a89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -50,17 +50,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedModels importedModels, AttributeFields attributeFields) { this.name = rankProfile.getName(); - compressedProperties = compress(removePartFromKeys(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields).derive())); - } - - private List<Pair<String, String>> removePartFromKeys(Map<String, String> map) { - ImmutableList.Builder<Pair<String, String>> replaced = new ImmutableList.Builder<>(); - for (Map.Entry<String, String> e : map.entrySet()) { - String key = e.getKey().replaceFirst(".part\\d+$", ""); - String val = e.getValue(); - replaced.add(new Pair<>(key, val)); - } - return replaced.build(); + compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields).derive()); } private Compressor.Compression compress(List<Pair<String, String>> properties) { @@ -185,57 +175,57 @@ public class RawRankProfile implements RankProfilesConfig.Producer { rankScoreDropLimit = rankProfile.getRankScoreDropLimit(); ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures(); rankProperties = new ArrayList<>(rankProfile.getRankProperties()); - derivePropertiesAndSummaryFeaturesFromMacros(rankProfile.getMacros()); + derivePropertiesAndSummaryFeaturesFromFunctions(rankProfile.getFunctions()); } - private void derivePropertiesAndSummaryFeaturesFromMacros(Map<String, RankProfile.Macro> macros) { - if (macros.isEmpty()) return; - Map<String, ExpressionFunction> expressionMacros = new LinkedHashMap<>(); - for (Map.Entry<String, RankProfile.Macro> macro : macros.entrySet()) { - expressionMacros.put(macro.getKey(), macro.getValue().asExpressionFunction()); + private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions) { + if (functions.isEmpty()) return; + Map<String, ExpressionFunction> expressionFunctions = new LinkedHashMap<>(); + for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : functions.entrySet()) { + expressionFunctions.put(function.getKey(), function.getValue().function()); } - Map<String, String> macroProperties = new LinkedHashMap<>(); - macroProperties.putAll(deriveMacroProperties(expressionMacros)); + Map<String, String> functionProperties = new LinkedHashMap<>(); + functionProperties.putAll(deriveFunctionProperties(expressionFunctions)); if (firstPhaseRanking != null) { - macroProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values()))); + functionProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values()))); } if (secondPhaseRanking != null) { - macroProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values()))); + functionProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values()))); } - for (Map.Entry<String, String> e : macroProperties.entrySet()) { + + for (Map.Entry<String, String> e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); } - SerializationContext context = new SerializationContext(expressionMacros.values(), null, macroProperties); - replaceMacroSummaryFeatures(context); + SerializationContext context = new SerializationContext(expressionFunctions.values(), null, functionProperties); + replaceFunctionSummaryFeatures(context); } - private Map<String, String> deriveMacroProperties(Map<String, ExpressionFunction> eMacros) { - SerializationContext context = new SerializationContext(eMacros); - for (Map.Entry<String, ExpressionFunction> e : eMacros.entrySet()) { + private Map<String, String> deriveFunctionProperties(Map<String, ExpressionFunction> functions) { + SerializationContext context = new SerializationContext(functions); + for (Map.Entry<String, ExpressionFunction> e : functions.entrySet()) { String expression = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expression); - } return context.serializedFunctions(); } - private void replaceMacroSummaryFeatures(SerializationContext context) { + private void replaceFunctionSummaryFeatures(SerializationContext context) { if (summaryFeatures == null) return; - Map<String, ReferenceNode> macroSummaryFeatures = new LinkedHashMap<>(); + Map<String, ReferenceNode> functionSummaryFeatures = new LinkedHashMap<>(); for (Iterator<ReferenceNode> i = summaryFeatures.iterator(); i.hasNext(); ) { ReferenceNode referenceNode = i.next(); - // Is the feature a macro? + // Is the feature a function? if (context.getFunction(referenceNode.getName()) != null) { context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()), referenceNode.toString(new StringBuilder(), context, null, null).toString()); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); - macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode); + functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode); i.remove(); // Will add the expanded one in next block } } - // Then, replace the summary features that were macros - for (Map.Entry<String, ReferenceNode> e : macroSummaryFeatures.entrySet()) { + // Then, replace the summary features that were functions + for (Map.Entry<String, ReferenceNode> e : functionSummaryFeatures.entrySet()) { summaryFeatures.add(e.getValue()); } } @@ -300,17 +290,12 @@ public class RawRankProfile implements RankProfilesConfig.Producer { return settings; } - /** - * Derives the properties this produces. Equal keys are suffixed with .part0 etc, remove when exporting to file - * - * @return map of the derived properties - */ - public Map<String, String> derive() { - Map<String, String> properties = new LinkedHashMap<>(); - int i = 0; + /** Derives the properties this produces */ + public List<Pair<String, String>> derive() { + List<Pair<String, String>> properties = new ArrayList<>(); for (RankProfile.RankProperty property : rankProperties) { if ("rankingExpression(firstphase).rankingScript".equals(property.getName())) { - // Could have been set by macro expansion. Set expressions, then skip this property. + // Could have been set by function expansion. Set expressions, then skip this property. try { firstPhaseRanking = new RankingExpression(property.getValue()); } catch (ParseException e) { @@ -325,100 +310,92 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } } else { - properties.put(property.getName() + ".part" + i, property.getValue()); - i++; + properties.add(new Pair<>(property.getName(), property.getValue())); } } - properties.putAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase")); - properties.putAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase")); + properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase")); + properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase")); for (FieldRankSettings settings : fieldRankSettings.values()) { - properties.putAll(settings.deriveRankProperties(i)); + properties.addAll(settings.deriveRankProperties()); } - i = 0; for (RankProfile.RankProperty property : boostAndWeightRankProperties) { - properties.put(property.getName() + ".part" + i, property.getValue()); - i++; + properties.add(new Pair<>(property.getName(), property.getValue())); } - i = 0; for (ReferenceNode feature : summaryFeatures) { - properties.put(summaryFeatureFefPropertyPrefix + ".part" + i, feature.toString()); - i++; + properties.add(new Pair<>(summaryFeatureFefPropertyPrefix, feature.toString())); } - i = 0; for (ReferenceNode feature : rankFeatures) { - properties.put(rankFeatureFefPropertyPrefix + ".part" + i, feature.toString()); - i++; + properties.add(new Pair<>(rankFeatureFefPropertyPrefix, feature.toString())); } if (numThreadsPerSearch > 0) { - properties.put("vespa.matching.numthreadspersearch", numThreadsPerSearch + ""); + properties.add(new Pair<>("vespa.matching.numthreadspersearch", numThreadsPerSearch + "")); } if (minHitsPerThread > 0) { - properties.put("vespa.matching.minhitsperthread", minHitsPerThread + ""); + properties.add(new Pair<>("vespa.matching.minhitsperthread", minHitsPerThread + "")); } if (numSearchPartitions >= 0) { - properties.put("vespa.matching.numsearchpartitions", numSearchPartitions + ""); + properties.add(new Pair<>("vespa.matching.numsearchpartitions", numSearchPartitions + "")); } if (termwiseLimit < 1.0) { - properties.put("vespa.matching.termwise_limit", termwiseLimit + ""); + properties.add(new Pair<>("vespa.matching.termwise_limit", termwiseLimit + "")); } if (matchPhaseSettings != null) { - properties.put("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute()); - properties.put("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + ""); - properties.put("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + ""); - properties.put("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + ""); - properties.put("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + ""); - properties.put("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + ""); + properties.add(new Pair<>("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "")); RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity(); if (diversitySettings != null) { - properties.put("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute()); - properties.put("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups())); - properties.put("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor())); - properties.put("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy())); + properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); } } if (rerankCount > -1) { - properties.put("vespa.hitcollector.heapsize", rerankCount + ""); + properties.add(new Pair<>("vespa.hitcollector.heapsize", rerankCount + "")); } if (keepRankCount > -1) { - properties.put("vespa.hitcollector.arraysize", keepRankCount + ""); + properties.add(new Pair<>("vespa.hitcollector.arraysize", keepRankCount + "")); } if (rankScoreDropLimit > -Double.MAX_VALUE) { - properties.put("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + ""); + properties.add(new Pair<>("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "")); } if (ignoreDefaultRankFeatures) { - properties.put("vespa.dump.ignoredefaultfeatures", String.valueOf(true)); + properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); } Iterator filterFieldsIterator = filterFields.iterator(); while (filterFieldsIterator.hasNext()) { String fieldName = (String) filterFieldsIterator.next(); - properties.put("vespa.isfilterfield." + fieldName + ".part42", String.valueOf(true)); + properties.add(new Pair<>("vespa.isfilterfield." + fieldName, String.valueOf(true))); } for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { - properties.put("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue()); + properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue())); } for (Map.Entry<String, String> queryFeatureType : queryFeatureTypes.entrySet()) { - properties.put("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue()); + properties.add(new Pair<>("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue())); } if (properties.size() >= 1000000) throw new RuntimeException("Too many rank properties"); return properties; } - private Map<String, String> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { - Map<String, String> ret = new LinkedHashMap<>(); - if (expression == null) { - return ret; - } + private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { + List<Pair<String, String>> properties = new ArrayList<>(); + if (expression == null) return properties; + String name = expression.getName(); - if ("".equals(name)) { + if ("".equals(name)) name = phase; - } + if (expression.getRoot() instanceof ReferenceNode) { - ret.put("vespa.rank." + phase, expression.getRoot().toString()); + properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); } else { - ret.put("vespa.rank." + phase, "rankingExpression(" + name + ")"); - ret.put("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString()); + properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); + properties.add(new Pair<>("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString())); } - return ret; + return properties; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java index a639165d297..cbabfffb7a1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java @@ -28,8 +28,8 @@ public class ExpressionTransforms { new XgboostFeatureConverter(), new ConstantDereferencer(), new ConstantTensorTransformer(), - new MacroInliner(), - new MacroShadower(), + new FunctionInliner(), + new FunctionShadower(), new TensorTransformer(), new Simplifier()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java index 6aef39db4da..c15ef20a455 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java @@ -8,11 +8,11 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; /** - * Inlines macros in ranking expressions + * Inlines functions in ranking expressions * * @author bratseth */ -public class MacroInliner extends ExpressionTransformer<RankProfileTransformContext> { +public class FunctionInliner extends ExpressionTransformer<RankProfileTransformContext> { @Override public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { @@ -24,9 +24,9 @@ public class MacroInliner extends ExpressionTransformer<RankProfileTransformCont } private ExpressionNode transformFeatureNode(ReferenceNode feature, RankProfileTransformContext context) { - RankProfile.Macro macro = context.inlineMacros().get(feature.getName()); - if (macro == null) return feature; - return transform(macro.getRankingExpression().getRoot(), context); // inline recursively and return + RankProfile.RankingExpressionFunction rankingExpressionFunction = context.inlineFunctions().get(feature.getName()); + if (rankingExpressionFunction == null) return feature; + return transform(rankingExpressionFunction.function().getBody().getRoot(), context); // inline recursively and return } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java index 758d2b2a87d..74b6471d291 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java @@ -10,19 +10,19 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; /** - * Transforms function nodes to reference nodes if a macro shadows a built-in function. - * This has the effect of allowing macros to redefine built-in functions. - * Another effect is that we can more or less add built-in functions over time - * without fear of breaking existing users' macros with the same name. + * Transforms function nodes to reference nodes if a rank profile function shadows a built-in function. + * This has the effect of allowing rank profile functions to redefine built-in functions. + * Another effect is that we can add built-in functions over time + * without fear of breaking existing users' functions with the same name. * - * However, there is a (largish) caveat. If a user has a macro with a certain number + * However, there is a (largish) caveat. If a user has a function with a certain number * of arguments, and we add in a built-in function with a different arity, * this will cause parse errors as the Java parser gives precedence to * built-in functions. * * @author lesters */ -public class MacroShadower extends ExpressionTransformer<RankProfileTransformContext> { +public class FunctionShadower extends ExpressionTransformer<RankProfileTransformContext> { @Override public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) { @@ -43,16 +43,14 @@ public class MacroShadower extends ExpressionTransformer<RankProfileTransformCon private ExpressionNode transformFunctionNode(FunctionNode function, RankProfileTransformContext context) { String name = function.getFunction().toString(); - RankProfile.Macro macro = context.rankProfile().getMacros().get(name); - if (macro == null) { + RankProfile.RankingExpressionFunction rankingExpressionFunction = context.rankProfile().getFunctions().get(name); + if (rankingExpressionFunction == null) { return transformChildren(function, context); } int functionArity = function.getFunction().arity(); - int macroArity = macro.getFormalParams() != null ? macro.getFormalParams().size() : 0; - if (functionArity != macroArity) { + if (functionArity != rankingExpressionFunction.function().arguments().size()) return transformChildren(function, context); - } ReferenceNode node = new ReferenceNode(name, function.children(), null); return transformChildren(node, context); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java index 40c3b997daa..2fe2dacf2ce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java @@ -20,25 +20,25 @@ public class RankProfileTransformContext extends TransformContext { private final RankProfile rankProfile; private final QueryProfileRegistry queryProfiles; private final ImportedModels importedModels; - private final Map<String, RankProfile.Macro> inlineMacros; + private final Map<String, RankProfile.RankingExpressionFunction> inlineFunctions; private final Map<String, String> rankProperties = new HashMap<>(); public RankProfileTransformContext(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedModels importedModels, Map<String, Value> constants, - Map<String, RankProfile.Macro> inlineMacros) { + Map<String, RankProfile.RankingExpressionFunction> inlineFunctions) { super(constants); this.rankProfile = rankProfile; this.queryProfiles = queryProfiles; this.importedModels = importedModels; - this.inlineMacros = inlineMacros; + this.inlineFunctions = inlineFunctions; } public RankProfile rankProfile() { return rankProfile; } public QueryProfileRegistry queryProfiles() { return queryProfiles; } public ImportedModels importedModels() { return importedModels; } - public Map<String, RankProfile.Macro> inlineMacros() { return inlineMacros; } + public Map<String, RankProfile.RankingExpressionFunction> inlineFunctions() { return inlineFunctions; } public Map<String, String> rankProperties() { return rankProperties; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java index cf123d0f7c1..fefb54a7fe3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java @@ -27,7 +27,7 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { search.allImportedFields() .flatMap(this::getSummaryFieldsForImportedField) .forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index e0d32ea8ccd..803a6c5ab40 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -26,7 +26,7 @@ public class AddExtraFieldsToDocument extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { SDDocumentType document = search.getDocument(); if (document != null) { for (Field field : search.extraFieldList()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java index 9ec596792fa..94589d94255 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java @@ -20,7 +20,7 @@ public class AttributeProperties extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { String fieldName = field.getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java index a95f4264dc6..23257e5eafd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java @@ -23,7 +23,7 @@ public class AttributesImplicitWord extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (fieldImplicitlyWordMatch(field)) { field.getMatching().setType(Matching.Type.WORD); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java index a3c4c97cf31..b9be30e8485 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java @@ -22,7 +22,7 @@ public class Bolding extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { for (SummaryField summary : field.getSummaryFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java index 37d60c1d32e..a0c4c8adb2d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; /** * Adds field sets for 1) fields defined inside document type 2) fields inside search but outside document + * * @author Vegard Havdal */ public class BuiltInFieldSets extends Processor { @@ -23,7 +24,7 @@ public class BuiltInFieldSets extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { addDocumentFieldSet(); addSearchFieldSet(); // "Hook" the field sets on search onto the document types, since we will include them diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java index d7b688be203..ad862ef767f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -38,7 +38,7 @@ public class CreatePositionZCurve extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { DataType fieldType = field.getDataType(); if ( ! isSupportedPositionType(fieldType)) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java index 7cc9b4e9b52..b34db6febd5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java @@ -15,7 +15,7 @@ public class DeprecateAttributePrefetch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java index 861ebad7085..076161a8584 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java @@ -23,7 +23,7 @@ public class DisallowComplexMapAndWsetKeyTypes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; // TODO also traverse struct types to search for bad map or wset types there. Do this after document manager is fixed, do diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java index 6b78da2146b..029892cba1c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java @@ -18,8 +18,9 @@ public class DiversitySettingsValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(search)) { if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index 59bc3dc66f4..a7c0ebd4a07 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -26,7 +26,7 @@ public class ExactMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { Matching.Type matching = field.getMatching().getType(); if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java index 9cfac625da5..cd33e4434e7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java @@ -21,7 +21,7 @@ public class FastAccessValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String invalidAttributes = search.allFields() @@ -29,7 +29,7 @@ public class FastAccessValidator extends Processor { .filter(FastAccessValidator::isIncompatibleAttribute) .map(Attribute::getName) .collect(Collectors.joining(", ")); - if (!invalidAttributes.isEmpty()) { + if ( ! invalidAttributes.isEmpty()) { throw new IllegalArgumentException( String.format( "For search '%s': The following attributes have a type that is incompatible with fast-access: %s. " + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java index 15c11589245..4ef9a9733d5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java @@ -24,7 +24,7 @@ public class FieldSetValidity extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (FieldSet fieldSet : search.fieldSets().userFieldSets().values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java index 0c75314ffa2..adb8ab62aab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java @@ -26,7 +26,9 @@ public class FilterFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + for (SDField f : search.allConcreteFields()) { if (f.getRanking().isFilter()) { filterField(f.getName()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index b51524b7e62..1f795458875 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -28,7 +28,7 @@ public class ImplicitSummaries extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { DocumentSummary defaultSummary = search.getSummary("default"); if (defaultSummary == null) { defaultSummary = new DocumentSummary("default"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java index 7464f574255..0d99c698aca 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java @@ -21,7 +21,7 @@ public class ImplicitSummaryFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary docsum : search.getSummaries().values()) { addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES), validate); addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES), validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index 9f03cdf4b5e..a3efd086c6a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -33,7 +33,7 @@ public class ImportedFieldsResolver extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { search.temporaryImportedFields().get().fields().forEach((name, field) -> resolveImportedField(field, validate)); search.setImportedFields(new ImportedFields(importedFields)); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java index 018183b91d8..210a8e7009c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java @@ -23,7 +23,7 @@ public class IndexFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java index 0740029de90..41355a76f47 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java @@ -24,7 +24,7 @@ public class IndexSettingsNonFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java index 419268468c2..aeab2bb6638 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java @@ -27,7 +27,7 @@ public class IndexingInputs extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java index 6f04184c512..11d69bf6c75 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java @@ -29,7 +29,7 @@ public class IndexingOutputs extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java index b73151768fd..27520647e3b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java @@ -25,7 +25,7 @@ public class IndexingValidation extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; VerificationContext context = new VerificationContext(new MyAdapter(search)); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java index f3907ae68cb..72777b7dfb4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java @@ -22,7 +22,7 @@ public class IndexingValues extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (Field field : search.getDocument().fieldSet()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java index c119dc2660b..baaf145dbce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java @@ -30,7 +30,7 @@ public class IntegerIndex2Attribute extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.doesIndexing() && field.getDataType().getPrimitiveType() instanceof NumericDataType) { if (field.getIndex(field.getName()) != null diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java index 507a0e87cff..fe94ac9849f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java @@ -31,7 +31,7 @@ public class LiteralBoost extends Processor { /** Adds extra search fields and indices to express literal boosts */ @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { checkRankModifierRankType(search); addLiteralBoostsToFields(search); reduceFieldLiteralBoosts(search); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java index f853b99bbb2..0daf7265daa 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java @@ -25,7 +25,7 @@ public class MakeAliases extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { List<String> usedAliases = new ArrayList<>(); for (SDField field : search.allConcreteFields()) { for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java index 2c43a65da99..6f67c22d9d2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java @@ -34,7 +34,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { DocumentSummary defaultSummary=search.getSummary("default"); for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values() ) { if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java index fe584fa41c0..ff8a5c2eb0b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java @@ -30,7 +30,7 @@ public class MatchConsistency extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Map<String, Matching.Type> types = new HashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java index 479384e09ef..b1728b9bd89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java @@ -20,8 +20,9 @@ public class MatchPhaseSettingsValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(search)) { RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java index 45e018f8fc3..a52a8ab74e6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java @@ -31,7 +31,7 @@ public class MultifieldIndexHarmonizer extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { populateIndexToFields(search); resolveAllConflicts(search); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java index 9bcb3929b3d..4d8f0032a78 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java @@ -15,12 +15,12 @@ public class MutableAttributes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { - if (!field.isExtraField() && field.getAttributes().containsKey(field.getName())) { + if ( ! field.isExtraField() && field.getAttributes().containsKey(field.getName())) { if (field.getAttributes().get(field.getName()).isMutable()) { throw new IllegalArgumentException("Field '" + field.getName() + "' in '" + search.getDocument().getName() + - "' can not be marked mutable as it is inside the document clause."); + "' can not be marked mutable as it is inside the document clause."); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java index cdfba54ee5a..3f3fc11380b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java @@ -26,7 +26,7 @@ public class NGramMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getMatching().getType().equals(Matching.Type.GRAM)) implementGramMatch(search, field, validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java index c89c709ffbf..8f2a29abcb6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java @@ -22,7 +22,7 @@ public class OptimizeIlscript extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java index 3583c4a0162..79f19efe422 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java @@ -33,7 +33,7 @@ public class PredicateProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { if (validate && field.doesIndexing()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 19025d37f8c..8c8c32389e2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -7,7 +7,9 @@ import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.processing.multifieldresolver.RankProfileTypeSettingsProcessor; import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; /** @@ -18,9 +20,7 @@ import java.util.List; */ public class Processing { - private static final List<ProcessorFactory> factories = createProcessorFactories(); - - private static List<ProcessorFactory> createProcessorFactories() { + private Collection<ProcessorFactory> processors() { return Arrays.asList( SearchMustHaveDocument::new, UrlFieldValidator::new, @@ -74,10 +74,9 @@ public class Processing { RankProfileTypeSettingsProcessor::new, ReferenceFieldsProcessor::new, FastAccessValidator::new, - ReservedMacroNames::new, + ReservedFunctionNames::new, RankingExpressionTypeValidator::new, - - // These should be last. + // These should be last: IndexingValidation::new, IndexingValues::new); } @@ -91,19 +90,18 @@ public class Processing { * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} * @param queryProfiles The query profiles contained in the application this search is part of. */ - public static void process(Search search, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles, - boolean validate) { + public void process(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) { + Collection<ProcessorFactory> factories = processors(); search.process(); factories.stream() .map(factory -> factory.create(search, deployLogger, rankProfileRegistry, queryProfiles)) - .forEach(processor -> processor.process(validate)); + .forEach(processor -> processor.process(validate, documentsOnly)); } @FunctionalInterface public interface ProcessorFactory { Processor create(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java index b938e40d9a2..6bfd0ef29ea 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java @@ -53,8 +53,10 @@ public abstract class Processor { * @param validate true to throw exceptions on validation errors, false to make the best possible effort * at completing processing without throwing an exception. * If we are not validating, emitting warnings have no effect and can (but must not) be skipped. + * @param documentsOnly true to skip processing (including validation, regardless of the validate setting) + * of aspects not relating to document definitions (e.g rank profiles) */ - public abstract void process(boolean validate); + public abstract void process(boolean validate, boolean documentsOnly); /** * Convenience method for adding a no-strings-attached implementation field for a regular field diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java index 81455991cc9..102d1910360 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java @@ -33,8 +33,9 @@ public class RankingExpressionTypeValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) { try { @@ -48,7 +49,6 @@ public class RankingExpressionTypeValidator extends Processor { /** Throws an IllegalArgumentException if the given rank profile does not produce valid type */ private void validate(RankProfile profile) { - profile.parseExpressions(); TypeContext context = profile.typeContext(queryProfiles); profile.getSummaryFeatures().forEach(f -> ensureValid(f, "summary feature " + f, context)); ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java index 76afcd5d520..0418538922b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java @@ -28,7 +28,7 @@ public class ReferenceFieldsProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { clearSummaryAttributeAspectForConcreteFields(); clearSummaryAttributeAspectForExplicitSummaryFields(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java index f2aa31bb9c3..805cbaced0f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java @@ -28,7 +28,7 @@ public class ReservedDocumentNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String docName = search.getDocument().getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java index adcebed9254..d7099215f17 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java @@ -13,29 +13,30 @@ import java.util.Set; import java.util.logging.Level; /** - * Issues a warning if some macro has a reserved name. This is not necessarily - * an error, as a macro can shadow a built-in function. + * Issues a warning if some function has a reserved name. This is not necessarily + * an error, as a rank profile function can shadow a built-in function. * * @author lesters */ -public class ReservedMacroNames extends Processor { +public class ReservedFunctionNames extends Processor { private static Set<String> reservedNames = getReservedNames(); - public ReservedMacroNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + public ReservedFunctionNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rp : rankProfileRegistry.all()) { - for (String macroName : rp.getMacros().keySet()) { - if (reservedNames.contains(macroName)) { - deployLogger.log(Level.WARNING, "Macro \"" + macroName + "\" " + - "in rank profile \"" + rp.getName() + "\" " + - "has a reserved name. This might mean that the macro shadows " + + for (String functionName : rp.getFunctions().keySet()) { + if (reservedNames.contains(functionName)) { + deployLogger.log(Level.WARNING, "Function '" + functionName + "' " + + "in rank profile '" + rp.getName() + "' " + + "has a reserved name. This might mean that the function shadows " + "the built-in function with the same name." ); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java index 403de1253b4..2d8eaff7762 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java @@ -19,7 +19,7 @@ public class SearchMustHaveDocument extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; if (search.getDocument() == null) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java index a0c884d25f9..8a4795c4dd2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java @@ -24,7 +24,8 @@ public class SetLanguage extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; List<String> textFieldsWithoutLanguage = new ArrayList<>(); for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java index a19ea8d7068..715828f808a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java @@ -20,7 +20,7 @@ public class SetRankTypeEmptyOnFilters extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getRanking().isFilter()) { field.setRankType(RankType.EMPTY); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java index 6426c724a07..defcf761649 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java @@ -21,7 +21,7 @@ public class SortingSettings extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java index d56b0272f06..e133f88f45c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java @@ -16,7 +16,7 @@ public class StringSettingsOnNonStringFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java index a952d3732b3..d2c4968ca26 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java @@ -26,7 +26,7 @@ public class SummaryConsistency extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary summary : search.getSummaries().values()) { if (summary.getName().equals("default")) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java index 647a433f201..b8d170c07f6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java @@ -26,7 +26,7 @@ public class SummaryDynamicStructsArrays extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java index 7bcbd9a267c..9b51c7c473e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java @@ -22,7 +22,7 @@ public class SummaryFieldsMustHaveValidSource extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (DocumentSummary summary : search.getSummaries().values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java index 23569cf39ae..678d5324e38 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java @@ -26,7 +26,7 @@ public class SummaryNamesFieldCollisions extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Map<String, Pair<String, String>> fieldToClassAndSource = new HashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java index 177fc7f2326..79b7a6067b9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java @@ -25,7 +25,7 @@ public class TagType extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag()) implementTagType(field); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 08571168336..8e54d7c00d6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -22,7 +22,7 @@ public class TensorFieldProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java index 645ed5121ea..74f30d6a730 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java @@ -34,7 +34,7 @@ public class TextMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getMatching().getType() != Matching.Type.TEXT) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index ac376982cfa..d81fdf70d20 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -28,7 +28,7 @@ public class UriHack extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.doesIndexing()) { DataType fieldType = field.getDataType(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java index ed813b42fff..c6b83349691 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java @@ -18,7 +18,7 @@ public class UrlFieldValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java index 54ad9f13f6f..21b7f1d2675 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java @@ -28,7 +28,7 @@ public class ValidateFieldTypes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String searchName = search.getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java index a0b204a25f2..408d60e1cff 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java @@ -21,7 +21,7 @@ public class ValidateFieldWithIndexSettingsCreatesIndex extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Matching defaultMatching = new Matching(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java index 892bcdad12c..13fe3f24d69 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java @@ -25,7 +25,7 @@ public class WordMatch extends Processor { super(search, deployLogger, rankProfileRegistry, queryProfiles); } - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if ( ! field.getMatching().getType().equals(Matching.Type.WORD)) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index cc1638347f6..ec4cbdfe58b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -22,7 +22,7 @@ import java.util.Map; import java.util.Optional; /** - * Class that processes a search instance and sets type settings on all rank profiles. + * This processes a search instance and sets type settings on all rank profiles. * * Currently, type settings are limited to the type of tensor attribute fields and tensor query features. * @@ -35,7 +35,9 @@ public class RankProfileTypeSettingsProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + processAttributeFields(); processImportedFields(); processQueryProfileTypes(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 282e5a29962..4b70b1b5ae2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -32,6 +32,7 @@ import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstants; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.vespa.model.ml.ConvertedModel; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModel; @@ -236,7 +237,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()), model.name(), profile, queryProfiles, model); for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) { - profile.addMacro(entry.getKey(), false).setRankingExpression(entry.getValue()); + profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false); } } } @@ -248,7 +249,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile); for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) { - profile.addMacro(entry.getKey(), false).setRankingExpression(entry.getValue()); + profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 3330910a797..16ef46353d9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; /** * This is the admin pseudo-plugin of the Vespa model, responsible for @@ -62,6 +63,11 @@ public class Admin extends AbstractConfigProducer implements Serializable { */ private ContainerCluster clusterControllers; + /** + * Cluster for container that might be running on logserver hosts + */ + private Optional<ContainerCluster> logServerContainerCluster = Optional.empty(); + private ZooKeepersConfigProvider zooKeepersConfigProvider; private FileDistributionConfigProducer fileDistribution; private final boolean multitenant; @@ -136,6 +142,12 @@ public class Admin extends AbstractConfigProducer implements Serializable { this.clusterControllers = clusterControllers; } + public Optional<ContainerCluster> getLogServerContainerCluster() { return logServerContainerCluster; } + + public void setLogserverContainerCluster(ContainerCluster logServerContainerCluster) { + this.logServerContainerCluster = Optional.of(logServerContainerCluster); + } + public ZooKeepersConfigProvider getZooKeepersConfigProvider() { return zooKeepersConfigProvider; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 25d184d148a..ea943f069cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -6,9 +6,7 @@ import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; -import com.yahoo.container.handler.LogHandler; import com.yahoo.log.LogLevel; -import com.yahoo.searchdefinition.derived.RankProfileList; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.admin.Admin; @@ -60,7 +58,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { NodesSpecification.optionalDedicatedFromParent(adminElement.getChild("logservers"), context); assignSlobroks(requestedSlobroks.orElse(NodesSpecification.nonDedicated(3, context)), admin); - assignLogserver(requestedLogservers.orElse(NodesSpecification.nonDedicated(1, context)), admin); + assignLogserver(requestedLogservers.orElse(createNodesSpecificationForLogserver()), admin); addLogForwarders(adminElement.getChild("logforwarding"), admin); } @@ -82,9 +80,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { if (hosts.isEmpty()) return; // No log server can be created (and none is needed) Logserver logserver = createLogserver(admin, hosts); - // TODO: Enable for main system as well - if (context.getDeployState().isHosted() && context.getDeployState().zone().system() == SystemName.cd) - createAdditionalContainerOnLogserverHost(admin, logserver.getHostResource()); + createAdditionalContainerOnLogserverHost(admin, logserver.getHostResource()); } else if (containerModels.iterator().hasNext()) { List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.count(), false); if (hosts.isEmpty()) return; // No log server can be created (and none is needed) @@ -95,25 +91,40 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { } } + private NodesSpecification createNodesSpecificationForLogserver() { + // TODO: Enable for main system as well + //if (context.getDeployState().isHosted() && context.getDeployState().zone().system() == SystemName.cd) + // return NodesSpecification.dedicated(1, context); + //else + return NodesSpecification.nonDedicated(1, context); + } + // Creates a container cluster 'logserver-cluster' with 1 container on logserver host // for setting up a handler for getting logs from logserver private void createAdditionalContainerOnLogserverHost(Admin admin, HostResource hostResource) { ContainerCluster logServerCluster = new ContainerCluster(admin, "logserver-cluster", "logserver-cluster"); ContainerModel logserverClusterModel = new ContainerModel(context.withParent(admin).withId(logServerCluster.getSubId())); - logserverClusterModel.setCluster(logServerCluster); + // Add base handlers and the log handler + logServerCluster.addMetricStateHandler(); + logServerCluster.addApplicationStatusHandler(); + logServerCluster.addStatisticsHandler(); + logServerCluster.addDefaultRootHandler(); addLogHandler(logServerCluster); - Container container = new Container(logServerCluster, "logserver-container", 0); + logserverClusterModel.setCluster(logServerCluster); + + Container container = new Container(logServerCluster, "" + 0, 0); container.setHostResource(hostResource); container.initService(); logServerCluster.addContainer(container); admin.addAndInitializeService(hostResource, container); + admin.setLogserverContainerCluster(logServerCluster); } private void addLogHandler(ContainerCluster cluster) { Handler<?> logHandler = Handler.fromClassName("com.yahoo.container.handler.LogHandler"); - logHandler.addServerBindings("http://*/logs/", "https://*/logs/"); + logHandler.addServerBindings("http://*/logs", "https://*/logs"); cluster.addComponent(logHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 94359e8672e..0ab0c0b6d4f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -123,6 +123,19 @@ public class NodesSpecification { Optional.empty()); } + /** Returns a requirement from <code>count</code> dedicated nodes in one group */ + public static NodesSpecification dedicated(int count, ConfigModelContext context) { + return new NodesSpecification(true, + count, + 1, + context.getDeployState().getWantedNodeVespaVersion(), + false, + ! context.getDeployState().getProperties().isBootstrap(), + false, + Optional.empty(), + Optional.empty()); + } + /** * Returns whether this requires dedicated nodes. * Otherwise the model encountering this request should reuse nodes requested for other purposes whenever possible. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java index 33f5edded3c..a9d3ec0e5a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java @@ -4,11 +4,10 @@ package com.yahoo.vespa.model.container.http; import com.yahoo.component.ComponentId; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator; -import com.yahoo.jdisc.http.ssl.DefaultSslTrustStoreConfigurator; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.component.SimpleComponent; +import com.yahoo.vespa.model.container.http.ssl.LegacySslProvider; import org.w3c.dom.Element; import static com.yahoo.component.ComponentSpecification.fromString; @@ -17,22 +16,23 @@ import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType; /** * @author Einar M R Rosenvinge * @author bjorncs + * @author mortent */ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig.Producer { private final String name; private final int listenPort; private final Element legacyConfig; + private final SimpleComponent sslProviderComponent; public ConnectorFactory(String name, int listenPort) { - this(name, listenPort, null, null, null); + this(name, listenPort, null, new LegacySslProvider(name)); } public ConnectorFactory(String name, int listenPort, Element legacyConfig, - Element sslKeystoreConfigurator, - Element sslTruststoreConfigurator) { + SimpleComponent sslProviderComponent) { super(new ComponentModel( new BundleInstantiationSpecification(new ComponentId(name), fromString("com.yahoo.jdisc.http.server.jetty.ConnectorFactory"), @@ -40,8 +40,9 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig this.name = name; this.listenPort = listenPort; this.legacyConfig = legacyConfig; - addSslKeyStoreConfigurator(name, sslKeystoreConfigurator); - addSslTrustStoreConfigurator(name, sslTruststoreConfigurator); + this.sslProviderComponent = sslProviderComponent; + addChild(sslProviderComponent); + inject(sslProviderComponent); } @Override @@ -49,6 +50,7 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig configureWithLegacyHttpConfig(legacyConfig, connectorBuilder); connectorBuilder.listenPort(listenPort); connectorBuilder.name(name); + ((ConnectorConfig.Producer)sslProviderComponent).getConfig(connectorBuilder); } public String getName() { @@ -152,31 +154,4 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig } } } - - private void addSslKeyStoreConfigurator(String name, Element sslKeystoreConfigurator) { - addSslConfigurator("ssl-keystore-configurator@" + name, - DefaultSslKeyStoreConfigurator.class, - sslKeystoreConfigurator); - } - - private void addSslTrustStoreConfigurator(String name, Element sslKeystoreConfigurator) { - addSslConfigurator("ssl-truststore-configurator@" + name, - DefaultSslTrustStoreConfigurator.class, - sslKeystoreConfigurator); - } - - private void addSslConfigurator(String idSpec, Class<?> defaultImplementation, Element configuratorElement) { - SimpleComponent configuratorComponent; - if (configuratorElement != null) { - String className = configuratorElement.getAttribute("class"); - String bundleName = configuratorElement.getAttribute("bundle"); - configuratorComponent = new SimpleComponent(new ComponentModel(idSpec, className, bundleName)); - } else { - configuratorComponent = - new SimpleComponent(new ComponentModel(idSpec, defaultImplementation.getName(), "jdisc_http_service")); - } - addChild(configuratorComponent); - inject(configuratorComponent); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/CustomSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/CustomSslProvider.java new file mode 100644 index 00000000000..bc211925576 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/CustomSslProvider.java @@ -0,0 +1,29 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http.ssl; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +import static com.yahoo.component.ComponentSpecification.fromString; + +/** + * @author mortent + */ +public class CustomSslProvider extends SimpleComponent implements ConnectorConfig.Producer { + public static final String COMPONENT_ID_PREFIX = "ssl-provider@"; + + public CustomSslProvider(String serverName, String className, String bundle) { + super(new ComponentModel( + new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX + serverName), + fromString(className), + fromString(bundle)))); + } + + @Override + public void getConfig(ConnectorConfig.Builder builder) { + builder.ssl.enabled(true); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DefaultSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DefaultSslProvider.java new file mode 100644 index 00000000000..3554ce95faf --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DefaultSslProvider.java @@ -0,0 +1,63 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http.ssl; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ssl.impl.DefaultSslContextFactoryProvider; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +import java.util.Optional; + +import static com.yahoo.component.ComponentSpecification.fromString; + +/** + * @author mortent + */ +public class DefaultSslProvider extends SimpleComponent implements ConnectorConfig.Producer { + public static final String COMPONENT_ID_PREFIX = "default-ssl-provider@"; + public static final String COMPONENT_CLASS = DefaultSslContextFactoryProvider.class.getName(); + public static final String COMPONENT_BUNDLE = "jdisc_http_service"; + + private final String privateKeyPath; + private final String certificatePath; + private final String caCertificatePath; + private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication; + + public DefaultSslProvider(String servername, String privateKeyPath, String certificatePath, String caCertificatePath, String clientAuthentication) { + super(new ComponentModel( + new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername), + fromString(COMPONENT_CLASS), + fromString(COMPONENT_BUNDLE)))); + this.privateKeyPath = privateKeyPath; + this.certificatePath = certificatePath; + this.caCertificatePath = caCertificatePath; + this.clientAuthentication = mapToConfigEnum(clientAuthentication); + } + + @Override + public void getConfig(ConnectorConfig.Builder builder) { + builder.ssl.enabled(true); + builder.ssl.privateKeyFile(privateKeyPath); + builder.ssl.certificateFile(certificatePath); + builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")); + builder.ssl.clientAuth(clientAuthentication); + } + + public SimpleComponent getComponent() { + return new SimpleComponent(new ComponentModel(getComponentId().stringValue(), COMPONENT_CLASS, COMPONENT_BUNDLE)); + } + + private static ConnectorConfig.Ssl.ClientAuth.Enum mapToConfigEnum(String clientAuthValue) { + if ("disabled".equals(clientAuthValue)) { + return ConnectorConfig.Ssl.ClientAuth.Enum.DISABLED; + } else if ("want".equals(clientAuthValue)) { + return ConnectorConfig.Ssl.ClientAuth.Enum.WANT_AUTH; + } else if ("need".equals(clientAuthValue)) { + return ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH; + } else { + return ConnectorConfig.Ssl.ClientAuth.Enum.DISABLED; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/LegacySslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/LegacySslProvider.java new file mode 100644 index 00000000000..7ab553c45b9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/LegacySslProvider.java @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http.ssl; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.impl.LegacySslContextFactoryProvider; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +import static com.yahoo.component.ComponentSpecification.fromString; + +/** + * Provides a legacy implementation of {@link SslContextFactoryProvider} to be injected into non-ssl connectors and connectors using legacy ssl config override + * + * @author bjorncs + */ +public class LegacySslProvider extends SimpleComponent implements ConnectorConfig.Producer { + + public static final String COMPONENT_ID_PREFIX = "legacy-ssl-provider@"; + public static final String COMPONENT_CLASS = LegacySslContextFactoryProvider.class.getName(); + public static final String COMPONENT_BUNDLE = "jdisc_http_service"; + + public LegacySslProvider(String serverName) { + super(new ComponentModel( + new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX + serverName), + fromString(COMPONENT_CLASS), + fromString(COMPONENT_BUNDLE)))); + } + + @Override + public void getConfig(ConnectorConfig.Builder builder) { + + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java index f88c091cd37..36736d66195 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -5,14 +5,20 @@ import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.text.XML; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; +import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider; +import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; +import com.yahoo.vespa.model.container.http.ssl.LegacySslProvider; import org.w3c.dom.Element; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; /** * @author Einar M R Rosenvinge + * @author mortent */ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuilder<ConnectorFactory> { private static final Logger log = Logger.getLogger(JettyConnectorBuilder.class.getName()); @@ -32,9 +38,31 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil legacyServerConfig = null; } } - Element sslKeystoreConfigurator = XML.getChild(serverSpec, "ssl-keystore-configurator"); - Element sslTruststoreConfigurator = XML.getChild(serverSpec, "ssl-truststore-configurator"); - return new ConnectorFactory(name, port, legacyServerConfig, sslKeystoreConfigurator, sslTruststoreConfigurator); + SimpleComponent sslProviderComponent = getSslConfigComponents(name, serverSpec); + return new ConnectorFactory(name, port, legacyServerConfig, sslProviderComponent); } + SimpleComponent getSslConfigComponents(String serverName, Element serverSpec) { + Element sslConfigurator = XML.getChild(serverSpec, "ssl"); + Element sslProviderConfigurator = XML.getChild(serverSpec, "ssl-provider"); + + if (sslConfigurator != null) { + String privateKeyFile = XML.getValue(XML.getChild(sslConfigurator, "private-key-file")); + String certificateFile = XML.getValue(XML.getChild(sslConfigurator, "certificate-file")); + Optional<String> caCertificateFile = XmlHelper.getOptionalChildValue(sslConfigurator, "ca-certificates-file"); + Optional<String> clientAuthentication = XmlHelper.getOptionalChildValue(sslConfigurator, "client-authentication"); + return new DefaultSslProvider( + serverName, + privateKeyFile, + certificateFile, + caCertificateFile.orElse(null), + clientAuthentication.orElse(null)); + } else if (sslProviderConfigurator != null) { + String className = sslProviderConfigurator.getAttribute("class"); + String bundle = sslProviderConfigurator.getAttribute("bundle"); + return new CustomSslProvider(serverName, className, bundle); + } else { + return new LegacySslProvider(serverName); + } + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index e2236feb336..adf5c81283e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -13,6 +13,7 @@ import com.yahoo.searchdefinition.FeatureNames; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; @@ -139,7 +140,7 @@ public class ConvertedModel { public ExpressionNode expression(FeatureArguments arguments, RankProfileTransformContext context) { RankingExpression expression = selectExpression(arguments); if (sourceModel.isPresent()) // we can verify - verifyRequiredMacros(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles()); + verifyRequiredFunctions(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles()); return expression.getRoot(); } @@ -183,41 +184,41 @@ public class ConvertedModel { QueryProfileRegistry queryProfiles, ModelStore store) { // Add constants - Set<String> constantsReplacedByMacros = new HashSet<>(); + Set<String> constantsReplacedByFunctions = new HashSet<>(); model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v)); model.largeConstants().forEach((k, v) -> transformLargeConstant(store, profile, queryProfiles, - constantsReplacedByMacros, k, v)); + constantsReplacedByFunctions, k, v)); - // Add macros - addGeneratedMacros(model, profile); + // Add functions + addGeneratedFunctions(model, profile); // Add expressions Map<String, RankingExpression> expressions = new HashMap<>(); for (Pair<String, RankingExpression> output : model.outputExpressions()) { addExpression(output.getSecond(), output.getFirst(), - constantsReplacedByMacros, + constantsReplacedByFunctions, model, store, profile, queryProfiles, expressions); } - // Transform and save macro - must come after reading expressions due to optimization transforms - // and must use the macro expression added to the profile, which may differ from the one saved in the model, + // Transform and save function - must come after reading expressions due to optimization transforms + // and must use the function expression added to the profile, which may differ from the one saved in the model, // after rewrite - model.macros().forEach((k, v) -> transformGeneratedMacro(store, constantsReplacedByMacros, k, - profile.getMacros().get(k).getRankingExpression())); + model.functions().forEach((k, v) -> transformGeneratedFunction(store, constantsReplacedByFunctions, k, + profile.getFunctions().get(k).function().getBody())); return expressions; } private static void addExpression(RankingExpression expression, String expressionName, - Set<String> constantsReplacedByMacros, + Set<String> constantsReplacedByFunctions, ImportedModel model, ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, Map<String, RankingExpression> expressions) { - expression = replaceConstantsByMacros(expression, constantsReplacedByMacros); + expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions); reduceBatchDimensions(expression, model, profile, queryProfiles); store.writeExpression(expressionName, expression); expressions.put(expressionName, expression); @@ -232,8 +233,8 @@ public class ConvertedModel { profile.rankingConstants().add(constant); } - for (Pair<String, RankingExpression> macro : store.readMacros()) { - addGeneratedMacroToProfile(profile, macro.getFirst(), macro.getSecond()); + for (Pair<String, RankingExpression> function : store.readFunctions()) { + addGeneratedFunctionToProfile(profile, function.getFirst(), function.getSecond()); } return store.readExpressions(); @@ -247,16 +248,16 @@ public class ConvertedModel { private static void transformLargeConstant(ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, - Set<String> constantsReplacedByMacros, + Set<String> constantsReplacedByFunctions, String constantName, Tensor constantValue) { - RankProfile.Macro macroOverridingConstant = profile.getMacros().get(constantName); - if (macroOverridingConstant != null) { - TensorType macroType = macroOverridingConstant.getRankingExpression().type(profile.typeContext(queryProfiles)); - if ( ! macroType.equals(constantValue.type())) - throw new IllegalArgumentException("Macro '" + constantName + "' replaces the constant with this name. " + - typeMismatchExplanation(constantValue.type(), macroType)); - constantsReplacedByMacros.add(constantName); // will replace constant(constantName) by constantName later + RankProfile.RankingExpressionFunction rankingExpressionFunctionOverridingConstant = profile.getFunctions().get(constantName); + if (rankingExpressionFunctionOverridingConstant != null) { + TensorType functionType = rankingExpressionFunctionOverridingConstant.function().getBody().type(profile.typeContext(queryProfiles)); + if ( ! functionType.equals(constantValue.type())) + throw new IllegalArgumentException("Function '" + constantName + "' replaces the constant with this name. " + + typeMismatchExplanation(constantValue.type(), functionType)); + constantsReplacedByFunctions.add(constantName); // will replace constant(constantName) by constantName later } else { Path constantPath = store.writeLargeConstant(constantName, constantValue); @@ -267,79 +268,75 @@ public class ConvertedModel { } } - private static void transformGeneratedMacro(ModelStore store, - Set<String> constantsReplacedByMacros, - String macroName, - RankingExpression expression) { + private static void transformGeneratedFunction(ModelStore store, + Set<String> constantsReplacedByFunctions, + String functionName, + RankingExpression expression) { - expression = replaceConstantsByMacros(expression, constantsReplacedByMacros); - store.writeMacro(macroName, expression); + expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions); + store.writeFunction(functionName, expression); } - private static void addGeneratedMacroToProfile(RankProfile profile, String macroName, RankingExpression expression) { - if (profile.getMacros().containsKey(macroName)) { - if ( ! profile.getMacros().get(macroName).getRankingExpression().equals(expression)) - throw new IllegalArgumentException("Generated macro '" + macroName + "' already exists in " + profile + + private static void addGeneratedFunctionToProfile(RankProfile profile, String functionName, RankingExpression expression) { + if (profile.getFunctions().containsKey(functionName)) { + if ( ! profile.getFunctions().get(functionName).function().getBody().equals(expression)) + throw new IllegalArgumentException("Generated function '" + functionName + "' already exists in " + profile + " - with a different definition" + - ": Has\n" + profile.getMacros().get(macroName).getRankingExpression() + + ": Has\n" + profile.getFunctions().get(functionName).function().getBody() + "\nwant to add " + expression + "\n"); return; } - RankProfile.Macro macro = profile.addMacro(macroName, false); // TODO: Inline if only used once - macro.setRankingExpression(expression); - macro.setTextualExpression(expression.getRoot().toString()); + profile.addFunction(new ExpressionFunction(functionName, expression), false); // TODO: Inline if only used once } /** - * Verify that the macros referred in the given expression exists in the given rank profile, - * and return tensors of the types specified in requiredMacros. + * Verify that the functions referred in the given expression exists in the given rank profile, + * and return tensors of the types specified in requiredFunctions. */ - private static void verifyRequiredMacros(RankingExpression expression, ImportedModel model, - RankProfile profile, QueryProfileRegistry queryProfiles) { - Set<String> macroNames = new HashSet<>(); - addMacroNamesIn(expression.getRoot(), macroNames, model); - for (String macroName : macroNames) { - TensorType requiredType = model.requiredMacros().get(macroName); - if (requiredType == null) continue; // Not a required macro - - RankProfile.Macro macro = profile.getMacros().get(macroName); - if (macro == null) - throw new IllegalArgumentException("Model refers input '" + macroName + - "' of type " + requiredType + " but this macro is not present in " + + private static void verifyRequiredFunctions(RankingExpression expression, ImportedModel model, + RankProfile profile, QueryProfileRegistry queryProfiles) { + Set<String> functionNames = new HashSet<>(); + addFunctionNamesIn(expression.getRoot(), functionNames, model); + for (String functionName : functionNames) { + TensorType requiredType = model.requiredFunctions().get(functionName); + if (requiredType == null) continue; // Not a required function + + RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName); + if (rankingExpressionFunction == null) + throw new IllegalArgumentException("Model refers input '" + functionName + + "' of type " + requiredType + " but this function is not present in " + profile); // TODO: We should verify this in the (function reference(s) this is invoked (starting from first/second // phase and summary features), as it may only resolve correctly given those bindings - // Or, probably better, annotate the macros with type constraints here and verify during general + // Or, probably better, annotate the functions with type constraints here and verify during general // type verification - TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles)); + TensorType actualType = rankingExpressionFunction.function().getBody().getRoot().type(profile.typeContext(queryProfiles)); if ( actualType == null) - throw new IllegalArgumentException("Model refers input '" + macroName + + throw new IllegalArgumentException("Model refers input '" + functionName + "' of type " + requiredType + - " which must be produced by a macro in the rank profile, but " + - "this macro references a feature which is not declared"); + " which must be produced by a function in the rank profile, but " + + "this function references a feature which is not declared"); if ( ! actualType.isAssignableTo(requiredType)) - throw new IllegalArgumentException("Model refers input '" + macroName + "'. " + + throw new IllegalArgumentException("Model refers input '" + functionName + "'. " + typeMismatchExplanation(requiredType, actualType)); } } private static String typeMismatchExplanation(TensorType requiredType, TensorType actualType) { - return "The required type of this is " + requiredType + ", but this macro returns " + actualType + + return "The required type of this is " + requiredType + ", but this function returns " + actualType + (actualType.rank() == 0 ? ". This is often due to missing declaration of query tensor features " + "in query profile types - see the documentation." : ""); } - /** - * Add the generated macros to the rank profile - */ - private static void addGeneratedMacros(ImportedModel model, RankProfile profile) { - model.macros().forEach((k, v) -> addGeneratedMacroToProfile(profile, k, v.copy())); + /** Add the generated functions to the rank profile */ + private static void addGeneratedFunctions(ImportedModel model, RankProfile profile) { + model.functions().forEach((k, v) -> addGeneratedFunctionToProfile(profile, k, v.copy())); } /** * Check if batch dimensions of inputs can be reduced out. If the input - * macro specifies that a single exemplar should be evaluated, we can + * function specifies that a single exemplar should be evaluated, we can * reduce the batch dimension out. */ private static void reduceBatchDimensions(RankingExpression expression, ImportedModel model, @@ -347,19 +344,19 @@ public class ConvertedModel { TypeContext<Reference> typeContext = profile.typeContext(queryProfiles); TensorType typeBeforeReducing = expression.getRoot().type(typeContext); - // Check generated macros for inputs to reduce - Set<String> macroNames = new HashSet<>(); - addMacroNamesIn(expression.getRoot(), macroNames, model); - for (String macroName : macroNames) { - if ( ! model.macros().containsKey(macroName)) continue; + // Check generated functions for inputs to reduce + Set<String> functionNames = new HashSet<>(); + addFunctionNamesIn(expression.getRoot(), functionNames, model); + for (String functionName : functionNames) { + if ( ! model.functions().containsKey(functionName)) continue; - RankProfile.Macro macro = profile.getMacros().get(macroName); - if (macro == null) { - throw new IllegalArgumentException("Model refers to generated macro '" + macroName + - "but this macro is not present in " + profile); + RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName); + if (rankingExpressionFunction == null) { + throw new IllegalArgumentException("Model refers to generated function '" + functionName + + "but this function is not present in " + profile); } - RankingExpression macroExpression = macro.getRankingExpression(); - macroExpression.setRoot(reduceBatchDimensionsAtInput(macroExpression.getRoot(), model, typeContext)); + RankingExpression functionExpression = rankingExpressionFunction.function().getBody(); + functionExpression.setRoot(reduceBatchDimensionsAtInput(functionExpression.getRoot(), model, typeContext)); } // Check expression for inputs to reduce @@ -378,7 +375,7 @@ public class ConvertedModel { List<ExpressionNode> children = ((TensorFunctionNode)node).children(); if (children.size() == 1 && children.get(0) instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode) children.get(0); - if (model.requiredMacros().containsKey(referenceNode.getName())) { + if (model.requiredFunctions().containsKey(referenceNode.getName())) { return reduceBatchDimensionExpression(tensorFunction, typeContext); } } @@ -386,7 +383,7 @@ public class ConvertedModel { } if (node instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode) node; - if (model.requiredMacros().containsKey(referenceNode.getName())) { + if (model.requiredFunctions().containsKey(referenceNode.getName())) { return reduceBatchDimensionExpression(TensorFunctionNode.wrapArgument(node), typeContext); } } @@ -447,47 +444,47 @@ public class ConvertedModel { } /** - * If a constant c is overridden by a macro, we need to replace instances of "constant(c)" by "c" in expressions. + * If a constant c is overridden by a function, we need to replace instances of "constant(c)" by "c" in expressions. * This method does that for the given expression and returns the result. */ - private static RankingExpression replaceConstantsByMacros(RankingExpression expression, - Set<String> constantsReplacedByMacros) { - if (constantsReplacedByMacros.isEmpty()) return expression; + private static RankingExpression replaceConstantsByFunctions(RankingExpression expression, + Set<String> constantsReplacedByFunctions) { + if (constantsReplacedByFunctions.isEmpty()) return expression; return new RankingExpression(expression.getName(), - replaceConstantsByMacros(expression.getRoot(), constantsReplacedByMacros)); + replaceConstantsByFunctions(expression.getRoot(), constantsReplacedByFunctions)); } - private static ExpressionNode replaceConstantsByMacros(ExpressionNode node, Set<String> constantsReplacedByMacros) { + private static ExpressionNode replaceConstantsByFunctions(ExpressionNode node, Set<String> constantsReplacedByFunctions) { if (node instanceof ReferenceNode) { Reference reference = ((ReferenceNode)node).reference(); if (FeatureNames.isSimpleFeature(reference) && reference.name().equals("constant")) { String argument = reference.simpleArgument().get(); - if (constantsReplacedByMacros.contains(argument)) + if (constantsReplacedByFunctions.contains(argument)) return new ReferenceNode(argument); } } if (node instanceof CompositeNode) { // not else: this matches some of the same nodes as the outer if above CompositeNode composite = (CompositeNode)node; return composite.setChildren(composite.children().stream() - .map(child -> replaceConstantsByMacros(child, constantsReplacedByMacros)) + .map(child -> replaceConstantsByFunctions(child, constantsReplacedByFunctions)) .collect(Collectors.toList())); } return node; } - private static void addMacroNamesIn(ExpressionNode node, Set<String> names, ImportedModel model) { + private static void addFunctionNamesIn(ExpressionNode node, Set<String> names, ImportedModel model) { if (node instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode)node; - if (referenceNode.getOutput() == null) { // macro references cannot specify outputs + if (referenceNode.getOutput() == null) { // function references cannot specify outputs names.add(referenceNode.getName()); - if (model.macros().containsKey(referenceNode.getName())) { - addMacroNamesIn(model.macros().get(referenceNode.getName()).getRoot(), names, model); + if (model.functions().containsKey(referenceNode.getName())) { + addFunctionNamesIn(model.functions().get(referenceNode.getName()).getRoot(), names, model); } } } else if (node instanceof CompositeNode) { for (ExpressionNode child : ((CompositeNode)node).children()) - addMacroNamesIn(child, names, model); + addFunctionNamesIn(child, names, model); } } @@ -551,19 +548,19 @@ public class ConvertedModel { return expressions; } - /** Adds this macro expression to the application package so it can be read later. */ - void writeMacro(String name, RankingExpression expression) { - application.getFile(modelFiles.macrosPath()).appendFile(name + "\t" + - expression.getRoot().toString() + "\n"); + /** Adds this function expression to the application package so it can be read later. */ + void writeFunction(String name, RankingExpression expression) { + application.getFile(modelFiles.functionsPath()).appendFile(name + "\t" + + expression.getRoot().toString() + "\n"); } - /** Reads the previously stored macro expressions for these arguments */ - List<Pair<String, RankingExpression>> readMacros() { + /** Reads the previously stored function expressions for these arguments */ + List<Pair<String, RankingExpression>> readFunctions() { try { - ApplicationFile file = application.getFile(modelFiles.macrosPath()); + ApplicationFile file = application.getFile(modelFiles.functionsPath()); if ( ! file.exists()) return Collections.emptyList(); - List<Pair<String, RankingExpression>> macros = new ArrayList<>(); + List<Pair<String, RankingExpression>> functions = new ArrayList<>(); BufferedReader reader = new BufferedReader(file.createReader()); String line; while (null != (line = reader.readLine())) { @@ -571,13 +568,13 @@ public class ConvertedModel { String name = parts[0]; try { RankingExpression expression = new RankingExpression(parts[0], parts[1]); - macros.add(new Pair<>(name, expression)); + functions.add(new Pair<>(name, expression)); } catch (ParseException e) { throw new IllegalStateException("Could not parse " + name, e); } } - return macros; + return functions; } catch (IOException e) { throw new UncheckedIOException(e); @@ -725,9 +722,9 @@ public class ConvertedModel { return storedModelReplicatedPath().append("constants"); } - /** Path to the macros file */ - public Path macrosPath() { - return storedModelReplicatedPath().append("macros.txt"); + /** Path to the functions file */ + public Path functionsPath() { + return storedModelReplicatedPath().append("functions.txt"); } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 63d3926afad..813d1d47533 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -34,6 +34,7 @@ import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferen import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.DocumentsOnlyRankProfile; import com.yahoo.searchdefinition.DefaultRankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; @@ -58,6 +59,8 @@ import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.search.query.ranking.Diversity; import java.util.Map; +import java.util.List; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.logging.Level; import org.apache.commons.lang.StringUtils; @@ -73,6 +76,7 @@ public class SDParser { private ApplicationPackage app; private DeployLogger deployLogger; private RankProfileRegistry rankProfileRegistry; + private boolean documentsOnly; /** For testing only */ public SDParser(String input, DeployLogger deployLogger) { @@ -81,17 +85,24 @@ public class SDParser { /** For testing only */ public SDParser(SimpleCharStream stream, DeployLogger deployLogger) { - this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry()); + this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry(), false); } + /** + * Creates a parser + * + * @param documentsOnly true to only parse the document aspect of a search definition (e.g skip rank profiles) + */ public SDParser(SimpleCharStream stream, DeployLogger deployLogger, ApplicationPackage applicationPackage, - RankProfileRegistry rankProfileRegistry) { + RankProfileRegistry rankProfileRegistry, + boolean documentsOnly) { this(stream); this.deployLogger = deployLogger; this.app = applicationPackage; this.rankProfileRegistry = rankProfileRegistry; + this.documentsOnly = documentsOnly; } /** @@ -1804,6 +1815,7 @@ void rankingConstant(Search search) : } lbrace() (rankingConstantItem(constant) (<NL>)*)+ <RBRACE> ) { + if (documentsOnly) return; search.rankingConstants().add(constant); } } @@ -1859,7 +1871,10 @@ void rankProfile(Search search) : { ( <RANKPROFILE> name = identifier() { - if ("default".equals(name)) { + if (documentsOnly) { + profile = new DocumentsOnlyRankProfile(name, search, rankProfileRegistry); + } + else if ("default".equals(name)) { profile = rankProfileRegistry.get(search, "default"); } else { profile = new RankProfile(name, search, rankProfileRegistry); @@ -1868,6 +1883,7 @@ void rankProfile(Search search) : [inheritsRankProfile(profile)] lbrace() (rankProfileItem(profile) (<NL>)*)* <RBRACE> ) { + if (documentsOnly) return; rankProfileRegistry.add(profile); } } @@ -1885,7 +1901,7 @@ Object rankProfileItem(RankProfile profile) : { } | fieldRankFilter(profile) | firstPhase(profile) | matchPhase(profile) - | macro(profile) + | function(profile) | ignoreRankFeatures(profile) | numThreadsPerSearch(profile) | minHitsPerThread(profile) @@ -1910,7 +1926,8 @@ void inheritsRankProfile(RankProfile profile) : String str; } { - <INHERITS> str = identifier() { profile.setInherited(str); } + <INHERITS> str = identifier() + { profile.setInherited(str); } } /** @@ -1918,19 +1935,20 @@ void inheritsRankProfile(RankProfile profile) : * * @param profile The profile to modify. */ -void macro(RankProfile profile) : +void function(RankProfile profile) : { - String macro, param, expr; + String name, expression, parameter; + List parameters = new ArrayList(); boolean inline = false; } { - ( <MACRO> inline = inline() macro = identifier() [ "$" { macro = macro + token.image; } ] - "(" { profile.addMacro(macro, inline); } - [ param = identifier() { profile.getMacros().get(macro).addParam(param); } - ( <COMMA> param = identifier() { profile.getMacros().get(macro).addParam(param); } )* ] + ( ( <FUNCTION> | <MACRO> ) inline = inline() name = identifier() [ "$" { name = name + token.image; } ] + "(" + [ parameter = identifier() { parameters.add(parameter); } + ( <COMMA> parameter = identifier() { parameters.add(parameter); } )* ] ")" - lbrace() expr = expression() (<NL>)* <RBRACE> ) - { profile.getMacros().get(macro).setTextualExpression(expr); } + lbrace() expression = expression() (<NL>)* <RBRACE> ) + { profile.addFunction(name, parameters, expression, inline); } } boolean inline() : @@ -2034,7 +2052,7 @@ Object firstPhaseItem(RankProfile profile) : double dropLimit; } { - ( expression = expression() { profile.setFirstPhaseRankingString(expression); } + ( expression = expression() { profile.setFirstPhaseRanking(expression); } | (<KEEPRANKCOUNT> <COLON> rerankCount = integer()) { profile.setKeepRankCount(rerankCount); } | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); } ) @@ -2063,7 +2081,7 @@ Object secondPhaseItem(RankProfile profile) : int rerankCount; } { - ( expression = expression() { profile.setSecondPhaseRankingString(expression); } + ( expression = expression() { profile.setSecondPhaseRanking(expression); } | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); } ) { return null; } diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 52d71574f82..4934ce113bb 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -63,8 +63,7 @@ Filtering = element filtering { HttpServer = element server { attribute port { xsd:nonNegativeInteger } & ComponentId & - element ssl-keystore-configurator { BundleSpec }? & # FOR INTERNAL USE ONLY - SUBJECT TO CHANGE - element ssl-truststore-configurator { BundleSpec }? & # FOR INTERNAL USE ONLY - SUBJECT TO CHANGE + (Ssl | SslProvider)? & GenericConfig* } @@ -90,6 +89,17 @@ ModelEvaluation = element model-evaluation { empty } +Ssl = element ssl { + element private-key-file { string } & + element certificate-file { string } & + element ca-certificates-file { string }? & + element client-authentication { string "disabled" | string "want" | string "need" }? +} + +SslProvider = element ssl-provider { + BundleSpec +} + # REST-API: RestApi = element rest-api { diff --git a/config-model/src/test/derived/gemini2/gemini.sd b/config-model/src/test/derived/gemini2/gemini.sd index 18be346a758..01e20c1b30a 100644 --- a/config-model/src/test/derived/gemini2/gemini.sd +++ b/config-model/src/test/derived/gemini2/gemini.sd @@ -6,19 +6,19 @@ search gemini { rank-profile test { - macro wrapper2(x) { + function wrapper2(x) { expression: x } - macro wrapper1(x) { + function wrapper1(x) { expression: wrapper2(x) } - macro toplevel() { + function toplevel() { expression: wrapper1(attribute(right)) } - macro interfering() { + function interfering() { expression: wrapper1(attribute(wrong)) } diff --git a/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd b/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd index 7f09095c5e7..6e399c03a2c 100644 --- a/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd +++ b/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd @@ -22,11 +22,11 @@ search rankexpression { } rank-profile macros { - macro titlematch$(var1, var2) { + function titlematch$(var1, var2) { expression: file: titlematch } - macro artistmatch() { + function artistmatch() { expression: 78+closeness(distance) } diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd index 96b0fa98098..0435ea439df 100644 --- a/config-model/src/test/examples/simple.sd +++ b/config-model/src/test/examples/simple.sd @@ -121,7 +121,7 @@ search simple { second-phase { rerank-count: 99 } - macro openTicket() { + function openTicket() { expression: if(attribute(status) == "accepted",1, if(attribute(status) == "new",1,if(attribute(status) == "reopened",1,0))) } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 85c75309d23..121863f454f 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -6,8 +6,13 @@ import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.search.core.ProtonConfig; @@ -16,6 +21,7 @@ import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.admin.Logserver; import com.yahoo.vespa.model.admin.Slobrok; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; @@ -927,6 +933,36 @@ public class ModelProvisioningTest { } @Test + public void testLogserverContainerWhenDedicatedLogserver() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'>" + + " <logservers>" + + " <nodes count='1' dedicated='true'/>" + + " </logservers>" + + " </admin>" + + " <container version='1.0' id='foo'>" + + " <nodes count='1'/>" + + " </container>" + + "</services>"; + testContainerOnLogserverHost(services); + } + + @Test + @Ignore // Ignore until we create container on logserver implicitly + public void testImplicitLogserverContainer() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <container version='1.0' id='foo'>" + + " <nodes count='1'/>" + + " </container>" + + "</services>"; + testContainerOnLogserverHost(services); + } + + @Test public void testUsingNodesAndGroupCountAttributesAndGettingTooFewNodes() { String services = "<?xml version='1.0' encoding='utf-8' ?>" + @@ -1712,4 +1748,29 @@ public class ModelProvisioningTest { return new ProtonConfig(builder); } + // Tests that a container is allocated on logserver host and that + // it is able to get config + private void testContainerOnLogserverHost(String services) { + int numberOfHosts = 2; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(numberOfHosts); + Zone zone = new Zone(SystemName.cd, Environment.prod, RegionName.defaultName()); + + VespaModel model = tester.createModel(zone, services, true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + Admin admin = model.getAdmin(); + Logserver logserver = admin.getLogserver(); + HostResource hostResource = logserver.getHostResource(); + assertNotNull(hostResource.getService("logserver")); + assertNotNull(hostResource.getService("container")); + + // Test that the container gets config + String configId = admin.getLogserver().getHostResource().getService("container").getConfigId(); + ApplicationMetadataConfig.Builder builder = new ApplicationMetadataConfig.Builder(); + model.getConfig(builder, configId); + ApplicationMetadataConfig cfg = new ApplicationMetadataConfig(builder); + assertEquals(1, cfg.generation()); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 03fa92f5cb9..07a36832094 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -5,10 +5,12 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.yolean.Exceptions; import org.junit.Test; import java.io.IOException; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -27,9 +29,9 @@ public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionT new DerivedConfiguration(search, registry, new QueryProfileRegistry(), new ImportedModels()); // cause rank profile parsing fail("parsing should have failed"); } catch (IllegalArgumentException e) { - e.printStackTrace(); - assertTrue(e.getCause().getMessage().contains("Could not read ranking expression file")); - assertTrue(e.getCause().getMessage().contains("wrongending.expr.expression")); + String message = Exceptions.toMessageString(e); + assertTrue(message.contains("Could not read ranking expression file")); + assertTrue(message.contains("wrongending.expr.expression")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java index 28559f351ac..02ec597c3ed 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java @@ -4,6 +4,8 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.Test; @@ -17,6 +19,7 @@ import static org.junit.Assert.assertNull; * @author Ulf Lilleengen */ public class RankProfileRegistryTest { + private static final String TESTDIR = "src/test/cfg/search/data/v2/inherited_rankprofiles"; @Test @@ -43,11 +46,11 @@ public class RankProfileRegistryTest { RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { - assertNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); - RankProfile rankProfileWithAddedMacro = new RankProfile(rankProfileName, search, rankProfileRegistry); - rankProfileWithAddedMacro.addMacro("foo", true); - rankProfileRegistry.add(rankProfileWithAddedMacro); - assertNotNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); + assertNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo")); + RankProfile rankProfileWithAddedFunction = new RankProfile(rankProfileName, search, rankProfileRegistry); + rankProfileWithAddedFunction.addFunction(new ExpressionFunction("foo", RankingExpression.from("1+2")), true); + rankProfileRegistry.add(rankProfileWithAddedFunction); + assertNotNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index a524a26cbef..150469cc928 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -59,7 +59,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " p2: 2.0 \n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: p2*p1\n" + " }\n" + " }\n" + @@ -76,7 +76,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase RankProfile child2 = rankProfileRegistry.get(s, "child2").compile(queryProfileRegistry, new ImportedModels()); assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString()); - assertEquals("foo: 14.0", child2.getMacros().get("foo").getRankingExpression().toString()); + assertEquals("foo: 14.0", child2.getFunctions().get("foo").function().getBody().toString()); List<Pair<String, String>> rankProperties = new RawRankProfile(child2, queryProfileRegistry, new ImportedModels(), @@ -101,7 +101,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " c: 7 \n" + " }\n" + - " macro c() {\n" + + " function c() {\n" + " expression: p2*p1\n" + " }\n" + " }\n" + @@ -114,7 +114,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase fail("Should have caused an exception"); } catch (IllegalArgumentException e) { - assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and macro named 'c'", + assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and function named 'c'", Exceptions.toMessageString(e)); } } @@ -132,7 +132,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro POP_SLOW_SCORE() {\n" + + " function POP_SLOW_SCORE() {\n" + " expression: safeLog(popShareSlowDecaySignal, -9.21034037)\n" + " }\n" + " }\n" + @@ -141,8 +141,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); - profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test @@ -161,7 +160,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " myValue: -9.21034037\n" + " }\n" + - " macro POP_SLOW_SCORE() {\n" + + " function POP_SLOW_SCORE() {\n" + " expression: safeLog(popShareSlowDecaySignal, myValue)\n" + " }\n" + " }\n" + @@ -170,14 +169,13 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); - profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test - public void testConstantDivisorInMacro() throws ParseException { + public void testConstantDivisorInFunction() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -186,7 +184,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro rank_default(){\n" + + " function rank_default(){\n" + " expression: k1 + (k2 + k3) / 100000000.0\n\n" + " }\n" + " }\n" + @@ -196,7 +194,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("k1 + (k2 + k3) / 100000000.0", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); } @Test @@ -212,7 +210,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro rank_default(){\n" + + " function rank_default(){\n" + " expression: 0.5+50*(attribute(rating_yelp)-3)\n\n" + " }\n" + " }\n" + @@ -222,7 +220,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index b13ffabda77..e507a6c48e4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -22,7 +22,7 @@ import static org.junit.Assert.fail; public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase { @Test - public void testMacroInliningPreserveArithemticOrdering() throws ParseException { + public void testFunctionInliningPreserveArithemticOrdering() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -44,18 +44,18 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " first-phase {\n" + " expression: p1 * add\n" + " }\n" + - " macro inline add() {\n" + + " function inline add() {\n" + " expression: 3 + attribute(a) + attribute(b) * mul3\n" + " }\n" + - " macro inline mul3() {\n" + + " function inline mul3() {\n" + " expression: attribute(a) * 3 + singleif\n" + " }\n" + - " macro inline singleif() {\n" + + " function inline singleif() {\n" + " expression: if (p1 < attribute(a), 1, 2) == 0\n" + " }\n" + " }\n" + " rank-profile child inherits parent {\n" + - " macro inline add() {\n" + + " function inline add() {\n" + " expression: 9 + attribute(a)\n" + " }\n" + " }\n" + @@ -95,7 +95,7 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " second-phase {\n" + " expression: p2 * foo\n" + " }\n" + - " macro inline foo() {\n" + + " function inline foo() {\n" + " expression: 3 + p1 + p2\n" + " }\n" + " }\n" + @@ -106,16 +106,16 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " constants {\n" + " p2: 2.0 \n" + " }\n" + - " macro bar() {\n" + + " function bar() {\n" + " expression: p2*p1\n" + " }\n" + - " macro inline baz() {\n" + + " function inline baz() {\n" + " expression: p2+p1+boz\n" + " }\n" + - " macro inline boz() {\n" + + " function inline boz() {\n" + " expression: 3.0\n" + " }\n" + - " macro inline arg(a1) {\n" + + " function inline arg(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -162,16 +162,16 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " first-phase {\n" + " expression: A + C + D\n" + " }\n" + - " macro inline D() {\n" + + " function inline D() {\n" + " expression: B + 1\n" + " }\n" + - " macro C() {\n" + + " function C() {\n" + " expression: A + B\n" + " }\n" + - " macro inline B() {\n" + + " function inline B() {\n" + " expression: attribute(b)\n" + " }\n" + - " macro inline A() {\n" + + " function inline A() {\n" + " expression: attribute(a)\n" + " }\n" + " }\n" + @@ -187,8 +187,8 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase } /** - * Expression evaluation has no stack so macro arguments are bound at config time creating a separate version of - * each macro for each binding, using hashes to name the bound variants of the macro. + * Expression evaluation has no stack so function arguments are bound at config time creating a separate version of + * each function for each binding, using hashes to name the bound variants of the function. * This method censors those hashes for string comparison. */ private String censorBindingHash(String s) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java index df9a40d29e2..17bebcba70e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java @@ -29,7 +29,7 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: foo\n" + " }\n" + " }\n" + @@ -61,10 +61,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: arg(5)\n" + " }\n" + - " macro arg(a1) {\n" + + " function arg(a1) {\n" + " expression: foo + a1*2\n" + " }\n" + " }\n" + @@ -96,10 +96,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: arg(foo)\n" + " }\n" + - " macro arg(a1) {\n" + + " function arg(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -131,10 +131,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(3)\n" + " }\n" + - " macro foo(a1) {\n" + + " function foo(a1) {\n" + " expression: bar(3)\n" + " }\n" + - " macro bar(a1) {\n" + + " function bar(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -159,10 +159,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(3)\n" + " }\n" + - " macro foo(a1) {\n" + + " function foo(a1) {\n" + " expression: bar(3) + bar(a1)\n" + " }\n" + - " macro bar(a1) {\n" + + " function bar(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -183,10 +183,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(bar(2))\n" + " }\n" + - " macro foo(x) {\n" + + " function foo(x) {\n" + " expression: x * x\n" + " }\n" + - " macro bar(x) {\n" + + " function bar(x) {\n" + " expression: x + x\n" + " }\n" + " }\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 1ece2355a92..e15d4075b19 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase { @Test - public void testBasicMacroShadowing() throws ParseException { + public void testBasicFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -31,7 +31,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -57,7 +57,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase @Test - public void testMultiLevelMacroShadowing() throws ParseException { + public void testMultiLevelFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -69,13 +69,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro tan(x) {\n" + + " function tan(x) {\n" + " expression: x * x\n" + " }\n" + - " macro cos(x) {\n" + + " function cos(x) {\n" + " expression: tan(x)\n" + " }\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: cos(x)\n" + " }\n" + " first-phase {\n" + @@ -113,7 +113,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase @Test - public void testMacroShadowingArguments() throws ParseException { + public void testFunctionShadowingArguments() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -125,7 +125,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -168,13 +168,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro relu(x) {\n" + // relu is a built in function, redefined here + " function relu(x) {\n" + // relu is a built in function, redefined here " expression: max(1.0, x)\n" + " }\n" + - " macro hidden_layer() {\n" + + " function hidden_layer() {\n" + " expression: relu(sum(query(q) * constant(W_hidden), input) + constant(b_input))\n" + " }\n" + - " macro final_layer() {\n" + + " function final_layer() {\n" + " expression: sigmoid(sum(hidden_layer * constant(W_final), hidden) + constant(b_final))\n" + " }\n" + " second-phase {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index 3fe3a7c3de1..5e649c2e551 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -5,9 +5,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.yolean.Exceptions; import org.junit.Ignore; import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -29,9 +31,7 @@ public class RankingExpressionValidationTestCase extends SearchDefinitionTestCas fail("No exception on incorrect ranking expression " + expression); } catch (IllegalArgumentException e) { // Success - // TODO: Where's the "com.yahoo.searchdefinition.parser.ParseException:" nonsense coming from? - assertTrue("Got unexpected error message: " + e.getCause().getMessage(), - e.getCause().getMessage().startsWith("com.yahoo.searchdefinition.parser.ParseException: Could not parse ranking expression '" + expression + "'")); + assertTrue(Exceptions.toMessageString(e).startsWith("Illegal first phase ranking function: Could not parse ranking expression '" + expression + "' in default, firstphase.:")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java index 8cdfdd51637..82c03c02f61 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java @@ -34,20 +34,19 @@ public class SearchImporterTestCase extends SearchDefinitionTestCase { assertEquals("simple",search.getName()); assertTrue(search.hasDocument()); - SDDocumentType document=search.getDocument(); - assertEquals("simple",document.getName()); - assertEquals(12,document.getFieldCount()); + SDDocumentType document = search.getDocument(); + assertEquals("simple", document.getName()); + assertEquals(12, document.getFieldCount()); SDField field; Attribute attribute; - new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true); + new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true, false); // First field field=(SDField) document.getField("title"); assertEquals(DataType.STRING,field.getDataType()); - assertEquals("{ summary | index; }", - field.getIndexingScript().toString()); + assertEquals("{ summary | index; }", field.getIndexingScript().toString()); assertTrue(!search.getIndex("default").isPrefix()); assertTrue(search.getIndex("title").isPrefix()); Iterator<String> titleAliases=search.getIndex("title").aliasIterator(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java index 274cf06aa37..f87a26c6f79 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java @@ -31,7 +31,7 @@ public class IdTestCase extends AbstractExportingTestCase { uri.parseIndexingScript("{ summary | index }"); document.addField(uri); - Processing.process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); assertNull(document.getField("uri")); assertNull(document.getField("Uri")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index 94d0bf6329a..2bae285301c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -41,7 +41,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { rankProfileRegistry.add(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); - Processing.process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels()); // Check attribute fields diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index 0e02970280e..ba19a8312f6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -69,14 +69,14 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase { assertTrue(!transforms.hasNext()); } @Test - public void testPositionDeriving() throws IOException, ParseException { + public void testPositionDeriving() { Search search = new Search("store", null); SDDocumentType document = new SDDocumentType("store"); search.addDocument(document); String fieldName = "location"; SDField field = document.addField(fieldName, PositionDataType.INSTANCE); field.parseIndexingScript("{ attribute | summary }"); - Processing.process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); SummaryMap summaryMap = new SummaryMap(search, new Summaries(search, new BaseDeployLogger())); Iterator transforms = summaryMap.resultTransformIterator(); @@ -141,7 +141,7 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase { } @Test - public void testFailOnSummaryFieldSourceCollision() throws IOException, ParseException { + public void testFailOnSummaryFieldSourceCollision() { try { Search search = SearchBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); } catch (Exception e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index 26b100a2d96..8941b07101d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -33,7 +33,7 @@ public class TypeConversionTestCase extends SearchDefinitionTestCase { a.parseIndexingScript("{ index }"); document.addField(a); - Processing.process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels()); IndexInfo indexInfo = derived.getIndexInfo(); assertFalse(indexInfo.hasCommand("default", "compact-to-term")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index a0556a156b5..48adc0eefc5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -36,7 +36,7 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest { AddAttributeTransformToSummaryOfImportedFields processor = new AddAttributeTransformToSummaryOfImportedFields( search,null,null,null); - processor.process(true); + processor.process(true, false); SummaryField summaryField = search.getSummaries().get(SUMMARY_NAME).getSummaryField(IMPORTED_FIELD_NAME); SummaryTransform actualTransform = summaryField.getTransform(); assertEquals(SummaryTransform.ATTRIBUTE, actualTransform); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java index 8f1cc9c3de8..3a0fedfd550 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java @@ -24,7 +24,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase { public void testInvalidAttributeProperties() throws IOException, ParseException { try { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties1.sd"); - new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); fail("attribute property should not be set"); } catch (RuntimeException e) { // empty @@ -34,7 +34,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase { @Test public void testValidAttributeProperties() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties2.sd"); - new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java index 4dee939e3da..1ab8b054cb7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -24,7 +24,7 @@ public class BoldingTestCase extends SearchDefinitionTestCase { public void testBoldingNonString() throws IOException, ParseException { try { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/processing/boldnonstring.sd"); - new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); fail(); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("'bolding: on' for non-text field")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java index 7230f176048..de08bf66548 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java @@ -188,7 +188,7 @@ public class ImportedFieldsResolverTestCase { private static ImportedFields resolve(Search search) { assertNotNull(search.temporaryImportedFields().get()); assertFalse(search.importedFields().isPresent()); - new ImportedFieldsResolver(search, null, null, null).process(true); + new ImportedFieldsResolver(search, null, null, null).process(true, false); assertFalse(search.temporaryImportedFields().isPresent()); assertNotNull(search.importedFields().get()); return search.importedFields().get(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java index 876e852aea1..e078d91f248 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java @@ -155,7 +155,7 @@ public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { sdoc.addField(unprocessedField); Search search = new Search("test", null); search.addDocument(sdoc); - Processing.process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); return unprocessedField.getIndexingScript(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java index 1e0270f293d..29bba224f46 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java @@ -19,11 +19,12 @@ import static org.junit.Assert.assertTrue; * @author baldersheim */ public class IntegerIndex2AttributeTestCase extends SearchDefinitionTestCase { + @Test public void testIntegerIndex2Attribute() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/integerindex2attribute.sd"); search.process(); - new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); SDField f; f = search.getConcreteField("s1"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index f67c85e2881..cff9abb08ed 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -72,9 +72,9 @@ class RankProfileSearchFixture { assertEquals(expValue, rankPropertyList.get(0).getValue()); } - public void assertMacro(String expexctedExpression, String macroName, String rankProfile) { + public void assertFunction(String expexctedExpression, String functionName, String rankProfile) { assertEquals(expexctedExpression, - compiledRankProfile(rankProfile).getMacros().get(macroName).getRankingExpression().getRoot().toString()); + compiledRankProfile(rankProfile).getFunctions().get(functionName).function().getBody().getRoot().toString()); } public RankProfile compileRankProfile(String rankProfile) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java index d8eb4368b57..0d8cbbf2e6a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java @@ -109,7 +109,7 @@ public class RankingExpressionTypeValidatorTestCase { } @Test - public void testMacroInvocationTypes() throws Exception { + public void testFunctionInvocationTypes() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString(joinLines( @@ -123,7 +123,7 @@ public class RankingExpressionTypeValidatorTestCase { " }", " }", " rank-profile my_rank_profile {", - " macro macro1(attribute_to_use) {", + " function macro1(attribute_to_use) {", " expression: attribute(attribute_to_use)", " }", " summary-features {", @@ -143,7 +143,7 @@ public class RankingExpressionTypeValidatorTestCase { } @Test - public void testTensorMacroInvocationTypes_Nested() throws Exception { + public void testTensorFunctionInvocationTypes_Nested() throws Exception { SearchBuilder builder = new SearchBuilder(); builder.importString(joinLines( "search test {", @@ -156,16 +156,16 @@ public class RankingExpressionTypeValidatorTestCase { " }", " }", " rank-profile my_rank_profile {", - " macro return_a() {", + " function return_a() {", " expression: return_first(attribute(a), attribute(b))", " }", - " macro return_b() {", + " function return_b() {", " expression: return_second(attribute(a), attribute(b))", " }", - " macro return_first(e1, e2) {", + " function return_first(e1, e2) {", " expression: e1", " }", - " macro return_second(e1, e2) {", + " function return_second(e1, e2) {", " expression: return_first(e2, e1)", " }", " summary-features {", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java index b046d60f948..8944409e1e9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -128,7 +128,7 @@ public class RankingExpressionWithOnnxTestCase { } @Test - public void testOnnxReferenceMissingMacro() throws ParseException { + public void testOnnxReferenceMissingFunction() throws ParseException { try { RankProfileSearchFixture search = new RankProfileSearchFixture( new StoringApplicationPackage(applicationDir), @@ -145,14 +145,14 @@ public class RankingExpressionWithOnnxTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + "onnx('mnist_softmax.onnx'): " + - "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " + "not present in rank profile 'my_profile'", Exceptions.toMessageString(expected)); } } @Test - public void testOnnxReferenceWithWrongMacroType() { + public void testOnnxReferenceWithWrongFunctionType() { try { RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d5[10])(0.0)", "onnx('mnist_softmax.onnx')"); @@ -163,7 +163,7 @@ public class RankingExpressionWithOnnxTestCase { assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + "onnx('mnist_softmax.onnx'): " + "Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " + - "but this macro returns tensor(d0[2],d5[10])", + "but this function returns tensor(d0[2],d5[10])", Exceptions.toMessageString(expected)); } } @@ -213,13 +213,13 @@ public class RankingExpressionWithOnnxTestCase { } @Test - public void testImportingFromStoredExpressionsWithMacroOverridingConstant() throws IOException { + public void testImportingFromStoredExpressionsWithFunctionOverridingConstant() throws IOException { String rankProfile = " rank-profile my_profile {\n" + - " macro Placeholder() {\n" + + " function Placeholder() {\n" + " expression: tensor(d0[2],d1[784])(0.0)\n" + " }\n" + - " macro " + name + "_Variable() {\n" + + " function " + name + "_Variable() {\n" + " expression: tensor(d1[10],d2[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -234,7 +234,7 @@ public class RankingExpressionWithOnnxTestCase { search.compileRankProfile("my_profile", applicationDir.append("models")); search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", search.search().rankingConstants().get( name + "_Variable")); // At this point the expression is stored - copy application to another location which do not have a models dir @@ -247,7 +247,7 @@ public class RankingExpressionWithOnnxTestCase { RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfile, storedApplication); searchFromStored.compileRankProfile("my_profile", applicationDir.append("models")); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", searchFromStored.search().rankingConstants().get( name + "_Variable")); } finally { IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); @@ -275,19 +275,19 @@ public class RankingExpressionWithOnnxTestCase { } } - private RankProfileSearchFixture fixtureWith(String macroExpression, + private RankProfileSearchFixture fixtureWith(String functionExpression, String firstPhaseExpression, String constant, String field, - String macroName, + String functionName, StoringApplicationPackage application) { try { RankProfileSearchFixture fixture = new RankProfileSearchFixture( application, application.getQueryProfiles(), " rank-profile my_profile {\n" + - " macro " + macroName + "() {\n" + - " expression: " + macroExpression + + " function " + functionName + "() {\n" + + " expression: " + functionExpression + " }\n" + " first-phase {\n" + " expression: " + firstPhaseExpression + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 14632a568ea..cba931e81f0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -160,7 +160,7 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testTensorFlowReferenceMissingMacro() throws ParseException { + public void testTensorFlowReferenceMissingFunction() throws ParseException { try { RankProfileSearchFixture search = new RankProfileSearchFixture( new StoringApplicationPackage(applicationDir), @@ -177,14 +177,14 @@ public class RankingExpressionWithTensorFlowTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + - "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " + "not present in rank profile 'my_profile'", Exceptions.toMessageString(expected)); } } @Test - public void testTensorFlowReferenceWithWrongMacroType() { + public void testTensorFlowReferenceWithWrongFunctionType() { try { RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d5[10])(0.0)", "tensorflow('mnist_softmax/saved')"); @@ -195,7 +195,7 @@ public class RankingExpressionWithTensorFlowTestCase { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + "Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " + - "but this macro returns tensor(d0[2],d5[10])", + "but this function returns tensor(d0[2],d5[10])", Exceptions.toMessageString(expected)); } } @@ -261,13 +261,13 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testImportingFromStoredExpressionsWithMacroOverridingConstantAndInheritance() throws IOException { + public void testImportingFromStoredExpressionsWithFunctionOverridingConstantAndInheritance() throws IOException { String rankProfiles = " rank-profile my_profile {\n" + - " macro Placeholder() {\n" + + " function Placeholder() {\n" + " expression: tensor(d0[2],d1[784])(0.0)\n" + " }\n" + - " macro " + name + "_layer_Variable_read() {\n" + + " function " + name + "_layer_Variable_read() {\n" + " expression: tensor(d1[10],d2[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -285,7 +285,7 @@ public class RankingExpressionWithTensorFlowTestCase { search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", search.search().rankingConstants().get("mnist_softmax_saved_layer_Variable_read")); // At this point the expression is stored - copy application to another location which do not have a models dir @@ -300,7 +300,7 @@ public class RankingExpressionWithTensorFlowTestCase { searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models")); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", searchFromStored.search().rankingConstants().get("mnist_softmax_saved_layer_Variable_read")); } finally { @@ -317,11 +317,11 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testMacroGeneration() { + public void testFunctionGeneration() { final String name = "mnist_saved"; - final String expression = "join(join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; - final String macroExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; - final String macroExpression2 = "join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; + final String expression = "join(join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; + final String functionExpression2 = "join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", "tensorflow('mnist/saved')", @@ -330,8 +330,8 @@ public class RankingExpressionWithTensorFlowTestCase { "input", new StoringApplicationPackage(applicationDir)); search.assertFirstPhaseExpression(expression, "my_profile"); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); } @Test @@ -339,7 +339,7 @@ public class RankingExpressionWithTensorFlowTestCase { final String name = "mnist_saved"; final String rankProfiles = " rank-profile my_profile {\n" + - " macro input() {\n" + + " function input() {\n" + " expression: tensor(d0[1],d1[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -349,9 +349,9 @@ public class RankingExpressionWithTensorFlowTestCase { " rank-profile my_profile_child inherits my_profile {\n" + " }"; - final String expression = "join(join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; - final String macroExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; - final String macroExpression2 = "join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; + final String expression = "join(join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; + final String functionExpression2 = "join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; RankProfileSearchFixture search = fixtureWithUncompiled(rankProfiles, new StoringApplicationPackage(applicationDir)); search.compileRankProfile("my_profile", applicationDir.append("models")); @@ -359,10 +359,10 @@ public class RankingExpressionWithTensorFlowTestCase { search.assertFirstPhaseExpression(expression, "my_profile"); search.assertFirstPhaseExpression(expression, "my_profile_child"); assertSmallConstant(name + "_dnn_hidden1_mul_x", TensorType.fromSpec("tensor()"), search); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile_child"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile_child"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile_child"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile_child"); // At this point the expression is stored - copy application to another location which do not have a models dir Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); @@ -377,10 +377,10 @@ public class RankingExpressionWithTensorFlowTestCase { searchFromStored.assertFirstPhaseExpression(expression, "my_profile"); searchFromStored.assertFirstPhaseExpression(expression, "my_profile_child"); assertSmallConstant(name + "_dnn_hidden1_mul_x", TensorType.fromSpec("tensor()"), search); - searchFromStored.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - searchFromStored.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile_child"); - searchFromStored.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); - searchFromStored.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile_child"); + searchFromStored.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + searchFromStored.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile_child"); + searchFromStored.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); + searchFromStored.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile_child"); } finally { IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); @@ -429,19 +429,19 @@ public class RankingExpressionWithTensorFlowTestCase { new StoringApplicationPackage(applicationDir)); } - private RankProfileSearchFixture fixtureWith(String macroExpression, + private RankProfileSearchFixture fixtureWith(String functionExpression, String firstPhaseExpression, String constant, String field, - String macroName, + String functionName, StoringApplicationPackage application) { try { RankProfileSearchFixture fixture = new RankProfileSearchFixture( application, application.getQueryProfiles(), " rank-profile my_profile {\n" + - " macro " + macroName + "() {\n" + - " expression: " + macroExpression + + " function " + functionName + "() {\n" + + " expression: " + functionExpression + " }\n" + " first-phase {\n" + " expression: " + firstPhaseExpression + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java index 0866d3192cf..48f2bf43e81 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java @@ -95,10 +95,10 @@ public class RankingExpressionWithTensorTestCase { } @Test - public void requireThatConstantTensorsCanBeUsedInMacro() throws ParseException { + public void requireThatConstantTensorsCanBeUsedInFunction() throws ParseException { RankProfileSearchFixture f = new RankProfileSearchFixture( " rank-profile my_profile {\n" + - " macro my_macro() {\n" + + " function my_macro() {\n" + " expression: sum(my_tensor)\n" + " }\n" + " first-phase {\n" + @@ -112,7 +112,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile"); - f.assertMacro("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); + f.assertFunction("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 86127a260c5..fd048737b43 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -20,24 +20,24 @@ import static org.junit.Assert.assertEquals; public class RankingExpressionsTestCase extends SearchDefinitionTestCase { @Test - public void testMacros() throws IOException, ParseException { + public void testFunctions() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction", rankProfileRegistry, new QueryProfileRegistry()).getSearch(); - final RankProfile macrosRankProfile = rankProfileRegistry.get(search, "macros"); - macrosRankProfile.parseExpressions(); - final Map<String, RankProfile.Macro> macros = macrosRankProfile.getMacros(); - assertEquals(2, macros.get("titlematch$").getFormalParams().size()); - assertEquals("var1", macros.get("titlematch$").getFormalParams().get(0)); - assertEquals("var2", macros.get("titlematch$").getFormalParams().get(1)); - assertEquals("var1 * var2 + 890", macros.get("titlematch$").getTextualExpression().trim()); - assertEquals("var1 * var2 + 890", macros.get("titlematch$").getRankingExpression().getRoot().toString()); - assertEquals("0.8+0.2*titlematch$(4,5)+0.8*titlematch$(7,8)*closeness(distance)", macrosRankProfile.getFirstPhaseRankingString().trim()); - assertEquals("78 + closeness(distance)", macros.get("artistmatch").getTextualExpression().trim()); - assertEquals(0, macros.get("artistmatch").getFormalParams().size()); + RankProfile functionsRankProfile = rankProfileRegistry.get(search, "macros"); + Map<String, RankProfile.RankingExpressionFunction> functions = functionsRankProfile.getFunctions(); + assertEquals(2, functions.get("titlematch$").function().arguments().size()); + assertEquals("var1", functions.get("titlematch$").function().arguments().get(0)); + assertEquals("var2", functions.get("titlematch$").function().arguments().get(1)); + assertEquals("var1 * var2 + 890", functions.get("titlematch$").function().getBody().getRoot().toString()); + assertEquals("0.8 + 0.2 * titlematch$(4,5) + 0.8 * titlematch$(7,8) * closeness(distance)", + functionsRankProfile.getFirstPhaseRanking().getRoot().toString()); + assertEquals("78 + closeness(distance)", + functions.get("artistmatch").function().getBody().getRoot().toString()); + assertEquals(0, functions.get("artistmatch").function().arguments().size()); - List<Pair<String, String>> rankProperties = new RawRankProfile(macrosRankProfile, + List<Pair<String, String>> rankProperties = new RawRankProfile(functionsRankProfile, new QueryProfileRegistry(), new ImportedModels(), new AttributeFields(search)).configProperties(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java index 8a07e99101c..b39c48b67bf 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java @@ -16,10 +16,10 @@ import static org.junit.Assert.assertTrue; /** * @author lesters */ -public class ReservedMacroNamesTestCase { +public class ReservedRankingExpressionFunctionNamesTestCase { @Test - public void requireThatMacrosWithReservedNamesIssueAWarning() throws ParseException { + public void requireThatFunctionsWithReservedNamesIssueAWarning() throws ParseException { TestDeployLogger deployLogger = new TestDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); @@ -32,10 +32,10 @@ public class ReservedMacroNamesTestCase { " }\n" + " \n" + " rank-profile test_rank_profile {\n" + - " macro not_a_reserved_name(x) {\n" + + " function not_a_reserved_name(x) {\n" + " expression: x + x\n" + " }\n" + - " macro sigmoid(x) {\n" + + " function sigmoid(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -43,7 +43,7 @@ public class ReservedMacroNamesTestCase { " }\n" + " }\n" + " rank-profile test_rank_profile_2 inherits test_rank_profile {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java index 85bee61c1b4..d0c1bf8b0ca 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -12,6 +12,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTestCase { @@ -20,41 +21,44 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTe Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidsummarysource.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage()); } } + @Test public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidimplicitsummarysource.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage()); } } + @Test public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidselfreferringsummary.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage()); } } + @Test public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/documentidinsummary.sd"); search.process(); BaseDeployLogger deployLogger = new BaseDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true); + new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true, false); assertEquals("documentid", search.getSummary("withid").getSummaryField("w").getSingleSource()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index 76c50821cb9..8e721dbe503 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -113,7 +113,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { } @Test - public void requireThatMaxAndMinWithTensorsReturnedFromMacrosAreReplaced() throws ParseException { + public void requireThatMaxAndMinWithTensorsReturnedFromFunctionsAreReplaced() throws ParseException { assertTransformedExpression("reduce(rankingExpression(returns_tensor),max,x)", "max(returns_tensor,x)"); assertTransformedExpression("reduce(rankingExpression(wraps_returns_tensor),max,x)", @@ -171,7 +171,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { " value: { {x:0}:0 }\n" + " }\n" + " }\n" + - " macro base_tensor() {\n" + + " function base_tensor() {\n" + " expression: constant(base_constant_tensor)\n" + " }\n" + " }\n" + @@ -181,19 +181,19 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { " value: { {x:0}:1 }\n" + " }\n" + " }\n" + - " macro returns_tensor_with_arg(arg1) {\n" + + " function returns_tensor_with_arg(arg1) {\n" + " expression: 2.0 * arg1\n" + " }\n" + - " macro wraps_returns_tensor() {\n" + + " function wraps_returns_tensor() {\n" + " expression: returns_tensor\n" + " }\n" + - " macro returns_tensor() {\n" + + " function returns_tensor() {\n" + " expression: attribute(tensor_field_2)\n" + " }\n" + - " macro tensor_inheriting() {\n" + + " function tensor_inheriting() {\n" + " expression: base_tensor\n" + " }\n" + - " macro testexpression() {\n" + + " function testexpression() {\n" + " expression: " + expression + "\n" + " }\n" + " }\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java index ab3bb113727..d0b6524a7e1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java @@ -40,7 +40,7 @@ public class ValidateFieldTypesTest { exceptionRule.expectMessage( "For search '" + DOCUMENT_NAME + "', field '" + IMPORTED_FIELD_NAME + "': Incompatible types. " + "Expected int for summary field '" + IMPORTED_FIELD_NAME + "', got string."); - validator.process(true); + validator.process(true, false); } private static Search createSearchWithDocument(String documentName) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index c1edbec6bf5..27839765930 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -78,7 +78,8 @@ public class DedicatedAdminV4Test { assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd"); assertHostContainsServices(model, "hosts/myhost1", "slobrok", "logd"); - assertHostContainsServices(model, "hosts/myhost2", "logserver", "logd"); + // Note: A container is always added on logserver host + assertHostContainsServices(model, "hosts/myhost2", "logserver", "logd", "container"); Monitoring monitoring = model.getAdmin().getMonitoring(); assertEquals("vespa.routing", monitoring.getClustername()); @@ -160,7 +161,8 @@ public class DedicatedAdminV4Test { assertHostContainsServices(model, "hosts/myhost0", "logd", "logforwarder", "slobrok"); assertHostContainsServices(model, "hosts/myhost1", "logd", "logforwarder", "slobrok"); - assertHostContainsServices(model, "hosts/myhost2", "logd", "logforwarder", "logserver"); + // Note: A container is always added on logserver host + assertHostContainsServices(model, "hosts/myhost2", "logd", "logforwarder", "logserver", "container"); Set<String> configIds = model.getConfigIds(); // 1 logforwarder on each host diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 54c4aabf44c..ceff9f3d4bb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -3,34 +3,30 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.container.ComponentsConfig; -import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.jdisc.FilterBindingsProvider; import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator; -import com.yahoo.jdisc.http.ssl.DefaultSslTrustStoreConfigurator; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.container.http.JettyHttpServer; +import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; import org.junit.Test; import org.w3c.dom.Element; -import org.xml.sax.SAXException; -import java.io.IOException; -import java.util.Arrays; import java.util.List; -import java.util.Set; +import java.util.Optional; import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author einarmr + * @author mortent */ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBase { @@ -192,60 +188,113 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas } @Test - public void ssl_keystore_and_truststore_configurator_can_be_overriden() throws IOException, SAXException { + public void verify_that_ssl_element_generates_connector_config_and_inject_provider_component() { Element clusterElem = DomBuilderTest.parse( "<jdisc id='default' version='1.0' jetty='true'>", - " <http>", - " <server port='9000' id='foo'>", - " <ssl-keystore-configurator class='com.yahoo.MySslKeyStoreConfigurator' bundle='mybundle'/>", - " <ssl-truststore-configurator class='com.yahoo.MySslTrustStoreConfigurator' bundle='mybundle'/>", - " </server>", - " <server port='9001' id='bar'/>", - " </http>", + " <http>", + " <server port='9000' id='minimal'>", + " <ssl>", + " <private-key-file>/foo/key</private-key-file>", + " <certificate-file>/foo/cert</certificate-file>", + " </ssl>", + " </server>", + " <server port='9001' id='with-cacerts'>", + " <ssl>", + " <private-key-file>/foo/key</private-key-file>", + " <certificate-file>/foo/cert</certificate-file>", + " <ca-certificates-file>/foo/cacerts</ca-certificates-file>", + " </ssl>", + " </server>", + " <server port='9002' id='need-client-auth'>", + " <ssl>", + " <private-key-file>/foo/key</private-key-file>", + " <certificate-file>/foo/cert</certificate-file>", + " <client-authentication>need</client-authentication>", + " </ssl>", + " </server>", + " </http>", nodesXml, + "", "</jdisc>"); + createModel(root, clusterElem); + ConnectorConfig minimalCfg = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/minimal/default-ssl-provider@minimal"); + assertTrue(minimalCfg.ssl().enabled()); + assertThat(minimalCfg.ssl().privateKeyFile(), is(equalTo("/foo/key"))); + assertThat(minimalCfg.ssl().certificateFile(), is(equalTo("/foo/cert"))); + assertThat(minimalCfg.ssl().caCertificateFile(), is(equalTo(""))); + assertThat(minimalCfg.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.DISABLED))); + + ConnectorConfig withCaCerts = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/with-cacerts/default-ssl-provider@with-cacerts"); + assertTrue(withCaCerts.ssl().enabled()); + assertThat(withCaCerts.ssl().privateKeyFile(), is(equalTo("/foo/key"))); + assertThat(withCaCerts.ssl().certificateFile(), is(equalTo("/foo/cert"))); + assertThat(withCaCerts.ssl().caCertificateFile(), is(equalTo("/foo/cacerts"))); + assertThat(withCaCerts.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.DISABLED))); + + ConnectorConfig needClientAuth = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/need-client-auth/default-ssl-provider@need-client-auth"); + assertTrue(needClientAuth.ssl().enabled()); + assertThat(needClientAuth.ssl().privateKeyFile(), is(equalTo("/foo/key"))); + assertThat(needClientAuth.ssl().certificateFile(), is(equalTo("/foo/cert"))); + assertThat(needClientAuth.ssl().caCertificateFile(), is(equalTo(""))); + assertThat(needClientAuth.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH))); + ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class); - { - ConnectorFactory firstConnector = connectorFactories.get(0); - assertConnectorHasInjectedComponents(firstConnector, "ssl-keystore-configurator@foo", "ssl-truststore-configurator@foo"); - assertComponentHasClassNameAndBundle(getChildComponent(firstConnector, 0), - "com.yahoo.MySslKeyStoreConfigurator", - "mybundle"); - assertComponentHasClassNameAndBundle(getChildComponent(firstConnector, 1), - "com.yahoo.MySslTrustStoreConfigurator", - "mybundle"); - } - { - ConnectorFactory secondConnector = connectorFactories.get(1); - assertConnectorHasInjectedComponents(secondConnector, "ssl-keystore-configurator@bar", "ssl-truststore-configurator@bar"); - assertComponentHasClassNameAndBundle(getChildComponent(secondConnector, 0), - DefaultSslKeyStoreConfigurator.class.getName(), - "jdisc_http_service"); - assertComponentHasClassNameAndBundle(getChildComponent(secondConnector, 1), - DefaultSslTrustStoreConfigurator.class.getName(), - "jdisc_http_service"); - } + connectorFactories.forEach(connectorFactory -> assertChildComponentExists(connectorFactory, DefaultSslProvider.COMPONENT_CLASS)); } - private static void assertConnectorHasInjectedComponents(ConnectorFactory connectorFactory, String... componentNames) { - Set<String> injectedComponentIds = connectorFactory.getInjectedComponentIds(); - assertThat(injectedComponentIds.size(), equalTo(componentNames.length)); - Arrays.stream(componentNames) - .forEach(name -> assertThat(injectedComponentIds, hasItem(name))); + @Test + public void verify_tht_ssl_provider_configuration_configures_correct_config() { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0' jetty='true'>", + " <http>", + " <server port='9000' id='ssl'>", + " <ssl-provider class='com.yahoo.CustomSslProvider' bundle='mybundle'/>", + " </server>", + " </http>", + nodesXml, + "", + "</jdisc>"); + + createModel(root, clusterElem); + ConnectorConfig sslProvider = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/ssl/ssl-provider@ssl"); + + assertTrue(sslProvider.ssl().enabled()); + + ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); + List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class); + ConnectorFactory connectorFactory = connectorFactories.get(0); + assertChildComponentExists(connectorFactory, "com.yahoo.CustomSslProvider"); } - private static SimpleComponent getChildComponent(ConnectorFactory connectorFactory, int index) { - return connectorFactory.getChildrenByTypeRecursive(SimpleComponent.class).get(index); + @Test + public void verify_that_container_factory_sees_same_config(){ + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0' jetty='true'>", + " <http>", + " <server port='9000' id='ssl'>", + " <ssl>", + " <private-key-file>/foo/key</private-key-file>", + " <certificate-file>/foo/cert</certificate-file>", + " </ssl>", + " </server>", + " </http>", + nodesXml, + "", + "</jdisc>"); + + createModel(root, clusterElem); + ConnectorConfig sslProvider = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/ssl"); + assertTrue(sslProvider.ssl().enabled()); } - private static void assertComponentHasClassNameAndBundle(SimpleComponent simpleComponent, - String className, - String bundleName) { - BundleInstantiationSpecification spec = simpleComponent.model.bundleInstantiationSpec; - assertThat(spec.classId.toString(), is(className)); - assertThat(spec.bundle.toString(), is(bundleName)); + private static void assertChildComponentExists(ConnectorFactory connectorFactory, String className) { + Optional<SimpleComponent> simpleComponent = connectorFactory.getChildren().values().stream() + .map(z -> (SimpleComponent) z) + .filter(component -> component.getClassId().stringValue().equals(className)) + .findFirst(); + assertTrue(simpleComponent.isPresent()); } private void assertJettyServerInConfig() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 8cc5144c2a3..1023733a652 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -14,6 +14,7 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; @@ -29,10 +30,10 @@ import java.util.Optional; * Helper class which sets up a system with multiple hosts. * Usage: * <code> - * VespaModelteser tester = new VespaModelTester(); + * VespaModelTester tester = new VespaModelTester(); * tester.addHosts(count, flavor); * ... add more nodes - * VesoaModel model = tester.createModel(servicesString); + * VespaModel model = tester.createModel(servicesString); * ... assert on model * </code> * @@ -93,12 +94,24 @@ public class VespaModelTester { /** Creates a model which uses 0 as start index and fails on out of capacity */ public VespaModel createModel(String services, String ... retiredHostNames) { - return createModel(services, true, retiredHostNames); + return createModel(Zone.defaultZone(), services, true, retiredHostNames); } + /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(services, failOnOutOfCapacity, 0, retiredHostNames); + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, 0, retiredHostNames); + } + + /** Creates a model which uses 0 as start index */ + public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, startIndexForClusters, retiredHostNames); } + + /** Creates a model which uses 0 as start index */ + public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { + return createModel(zone, services, failOnOutOfCapacity, 0, retiredHostNames); + } + /** * Creates a model using the hosts already added to this * @@ -107,7 +120,7 @@ public class VespaModelTester { * is available or if we should just silently receive a smaller allocation * @return the resulting model */ - public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { + public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; @@ -124,6 +137,7 @@ public class VespaModelTester { .applicationPackage(appPkg) .modelHostProvisioner(provisioner) .properties(properties) + .zone(zone) .build(); return modelCreatorWithMockPkg.create(false, deployState, configModelRegistry); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index 3e060b7a0dc..a58b6a2e372 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -17,18 +17,16 @@ import java.util.List; public class ApplicationPackageUtils { public static String generateSearchDefinition(String name, String field1, String field2) { - String sd = "" + + return "" + "search " + name + "{" + " document " + name + "{" + " field " + field1 + " type string {\n" + " indexing: index | summary\n" + " summary: dynamic\n" + - " header\n" + " }\n" + " field " + field2 + " type int {\n" + " indexing: attribute | summary\n" + " attribute: fast-access\n" + - " header\n" + " }\n" + " field " + field2 + "_nfa type int {\n" + " indexing: attribute \n" + @@ -48,7 +46,6 @@ public class ApplicationPackageUtils { " rank-features: attribute(" + field2 + ")" + " }" + "}"; - return sd; } public static Search createSearch(String name, String field1, String field2) throws ParseException { diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index e740e7d86b0..632abe68ab7 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -112,15 +112,23 @@ </request-chain> </filtering> - <server port="4080" id="myServer"> - <ssl-keystore-configurator class="com.yahoo.MySslKeyStoreConfigurator" bundle="mybundle" /> - <ssl-truststore-configurator class="com.yahoo.MySslTrustStoreConfigurator" bundle="mybundle" /> - </server> + <server port="4080" id="myServer"/> <server port="4081" id="anotherServer"> <config name="container.jdisc.config.http-server"> <maxChunkSize>9999</maxChunkSize> </config> </server> + <server port="4082" id="defaultSsl"> + <ssl> + <private-key-file>/foo/key</private-key-file> + <certificate-file>/foo/cert</certificate-file> + <ca-certificates-file>/foo/cacerts</ca-certificates-file> + <client-authentication>want</client-authentication> + </ssl> + </server> + <server port="4083" id="sslProvider"> + <ssl-provider class="com.yahoo.MySslProvider" bundle="mybundle"/> + </server> </http> <accesslog type='json' diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index b0c04cea1b0..78507779585 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -26,7 +26,7 @@ public class Flavor { private final String description; private final boolean retired; private List<Flavor> replacesFlavors; - private int idealHeadroom; + private int idealHeadroom; // Note: Not used after Vespa 6.282 /** * Creates a Flavor, but does not set the replacesFlavors. diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java deleted file mode 100644 index ca8d531634b..00000000000 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; - -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Inspector; -import com.yahoo.vespa.config.SlimeUtils; - -import java.util.LinkedHashSet; -import java.util.Optional; -import java.util.Set; - -/** - * @author bratseth - * @deprecated use AllocatedHosts - */ -// TODO: Remove when no version older than 6.143 is in production anywhere -@Deprecated -@SuppressWarnings("unused") -public class ProvisionInfo extends AllocatedHosts { - - private static final String mappingKey = "mapping"; - private static final String hostSpecKey = "hostSpec"; - - private ProvisionInfo(Set<HostSpec> hosts) { - super(hosts); - } - - public static ProvisionInfo withHosts(Set<HostSpec> hosts) { - return new ProvisionInfo(hosts); - } - - public static ProvisionInfo fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { - return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); - } - - private static ProvisionInfo fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) { - Inspector array = inspector.field(mappingKey); - Set<HostSpec> hosts = new LinkedHashSet<>(); - array.traverse(new ArrayTraverser() { - @Override - public void entry(int i, Inspector inspector) { - hosts.add(hostFromSlime(inspector.field(hostSpecKey), nodeFlavors)); - } - }); - return new ProvisionInfo(hosts); - } - -} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java index 334228333c5..ea2ce324a27 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java @@ -6,8 +6,7 @@ import java.util.Objects; /** * Represents a tenant in the provision API. * - * @author lulf - * @since 5.12 + * @author Ulf Lilleengen */ public class TenantName implements Comparable<TenantName> { diff --git a/config-provisioning/src/main/resources/configdefinitions/flavors.def b/config-provisioning/src/main/resources/configdefinitions/flavors.def index 57affc2f104..63b22958487 100644 --- a/config-provisioning/src/main/resources/configdefinitions/flavors.def +++ b/config-provisioning/src/main/resources/configdefinitions/flavors.def @@ -47,4 +47,5 @@ flavor[].description string default="" flavor[].retired bool default=false # The free capacity we would like to preserve for this flavor +# Note: Not used after Vespa 6.282 flavor[].idealHeadroom int default=0 diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml index 0985eeca6cf..e5498aae5ec 100644 --- a/config-proxy/pom.xml +++ b/config-proxy/pom.xml @@ -87,24 +87,31 @@ </configuration> </plugin> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - </configuration> - <executions> - <execution> - <id>make-assembly</id> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <finalName>${project.artifactId}-jar-with-dependencies</finalName> + <filters> + <filter> + <!-- Don't include signature files from bouncycastle in uber jar. --> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + </execution> + </executions> + </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java index ad99afe3f36..90532344a58 100644 --- a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java +++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java @@ -155,7 +155,7 @@ public class CfgConfigPayloadBuilder { } private boolean isArray(String name) { - return name.endsWith("]"); + return name.endsWith("]") && !name.startsWith("["); } private boolean isMap(String name) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 6a55fb77933..6c67f730f60 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -483,10 +483,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Logs ---------------------------------------------------------------- - public HttpResponse getLogs(ApplicationId applicationId) { - String logServerHostName = getLogServerURI(applicationId); + public HttpResponse getLogs(ApplicationId applicationId, String apiParams) { + String logServerURI = getLogServerURI(applicationId) + apiParams; LogRetriever logRetriever = new LogRetriever(); - return logRetriever.getLogs(logServerHostName); + return logRetriever.getLogs(logServerURI); } // ---------------- Session operations ---------------------------------------------------------------- diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java index dd60d158313..9e8bcdccf79 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java @@ -21,7 +21,7 @@ public class LogRetriever { try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) { org.apache.http.HttpResponse response = httpClient.execute(get); String responseBody = EntityUtils.toString(response.getEntity(), "UTF-8"); - return new HttpResponse(response.getStatusLine().getStatusCode()) { + return new LogsResponse(response.getStatusLine().getStatusCode()) { @Override public void render(OutputStream outputStream) throws IOException { if (response.getEntity() != null ) outputStream.write(responseBody.getBytes()); @@ -29,13 +29,26 @@ public class LogRetriever { }; } catch (IOException e) { log.log(Level.WARNING, "Failed to retrieve logs from log server", e); - return new HttpResponse(404) { + return new LogsResponse(404) { @Override public void render(OutputStream outputStream) throws IOException { outputStream.write(e.toString().getBytes()); } + }; } } + + private abstract static class LogsResponse extends HttpResponse { + + LogsResponse(int status) { + super(status); + } + + @Override + public String getContentType() { + return "application/json"; + } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index b65cb370f93..528575f4f27 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -97,7 +97,9 @@ public class ApplicationHandler extends HttpHandler { } if (isLogRequest(request)) { - return applicationRepository.getLogs(applicationId); + String apiParams = request.getUri().getQuery(); + apiParams = apiParams == null ? "" : "?" + apiParams; + return applicationRepository.getLogs(applicationId, apiParams); } return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId)); diff --git a/configserver/src/test/apps/app-logserver-with-container/hosts.xml b/configserver/src/test/apps/app-logserver-with-container/hosts.xml new file mode 100644 index 00000000000..d5a51f050fd --- /dev/null +++ b/configserver/src/test/apps/app-logserver-with-container/hosts.xml @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="localhost"> + <alias>node1</alias> + </host> +</hosts> + diff --git a/configserver/src/test/apps/app-logserver-with-container/services.xml b/configserver/src/test/apps/app-logserver-with-container/services.xml new file mode 100644 index 00000000000..3b88fc3879d --- /dev/null +++ b/configserver/src/test/apps/app-logserver-with-container/services.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1"/> + <logserver hostalias="node1"/> + </admin> + + + + <container version="1.0"> + <nodes> + <node hostalias="node1" /> + </nodes> + </container> + +</services> diff --git a/configserver/src/test/apps/app/services.xml b/configserver/src/test/apps/app/services.xml index 6cc30b8b6ec..457a3fad397 100644 --- a/configserver/src/test/apps/app/services.xml +++ b/configserver/src/test/apps/app/services.xml @@ -4,6 +4,7 @@ <admin version="2.0"> <adminserver hostalias="node1"/> + <logserver hostalias="node1" /> </admin> <content version="1.0"> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index a5e76262f48..2ae8917e905 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -1,6 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; @@ -13,6 +15,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; @@ -41,6 +44,11 @@ import java.util.Collections; import java.util.Optional; import java.util.Set; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -58,6 +66,7 @@ public class ApplicationRepositoryTest { private final static File testApp = new File("src/test/apps/app"); private final static File testAppJdiscOnly = new File("src/test/apps/app-jdisc-only"); private final static File testAppJdiscOnlyRestart = new File("src/test/apps/app-jdisc-only-restart"); + private final static File testAppLogServerWithContainer = new File("src/test/apps/app-logserver-with-container"); private final static TenantName tenant1 = TenantName.from("test1"); private final static TenantName tenant2 = TenantName.from("test2"); @@ -109,6 +118,27 @@ public class ApplicationRepositoryTest { } @Test + public void getLogs() { + WireMockServer wireMock = new WireMockServer(wireMockConfig().port(8080)); + wireMock.start(); + WireMock.configureFor("localhost", wireMock.port()); + stubFor(get(urlEqualTo("/logs")) + .willReturn(aResponse() + .withStatus(200))); + wireMock.start(); + deployApp(testAppLogServerWithContainer); + HttpResponse response = applicationRepository.getLogs(applicationId(), ""); + assertEquals(200, response.getStatus()); + wireMock.stop(); + } + + @Test(expected = IllegalArgumentException.class) + public void getLogsNoContainerOnLogServerHostShouldThrowException() { + deployApp(testApp); + applicationRepository.getLogs(applicationId(), ""); + } + + @Test public void deleteUnusedTenants() { // Set clock to epoch plus hour, as mock curator will always return epoch as creation time Instant now = ManualClock.at("1970-01-01T01:00:00"); diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 4183b642af1..4c12bacf145 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -10,11 +10,13 @@ import org.json.JSONObject; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.util.Arrays; +import java.util.HashMap; import java.util.concurrent.Executor; public class LogHandler extends ThreadedHttpRequestHandler { - private static final String LOG_DIRECTORY = "/home/y/logs/vespa/"; + private static final String LOG_DIRECTORY = "/home/y/logs/vespa/logarchive/"; @Inject public LogHandler(Executor executor) { @@ -23,10 +25,15 @@ public class LogHandler extends ThreadedHttpRequestHandler { @Override public HttpResponse handle(HttpRequest request) { - JSONObject logJson; + JSONObject responseJSON = new JSONObject(); + HashMap<String, String> apiParams = getParameters(request); + long earliestLogThreshold = getEarliestThreshold(apiParams); + long latestLogThreshold = getLatestThreshold(apiParams); + LogReader logReader= new LogReader(earliestLogThreshold, latestLogThreshold); try { - logJson = LogReader.readLogs(LOG_DIRECTORY); + JSONObject logJson = logReader.readLogs(LOG_DIRECTORY); + responseJSON.put("logs", logJson.toString()); } catch (IOException | JSONException e) { return new HttpResponse(404) { @Override @@ -37,9 +44,33 @@ public class LogHandler extends ThreadedHttpRequestHandler { @Override public void render(OutputStream outputStream) throws IOException { OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream); - outputStreamWriter.write(logJson.toString()); + outputStreamWriter.write(responseJSON.toString()); outputStreamWriter.close(); } }; } + + private HashMap<String, String> getParameters(HttpRequest request) { + String query = request.getUri().getQuery(); + HashMap<String, String> keyValPair = new HashMap<>(); + Arrays.stream(query.split("&")).forEach(pair -> { + String[] splitPair = pair.split("="); + keyValPair.put(splitPair[0], splitPair[1]); + }); + return keyValPair; + } + + private long getEarliestThreshold(HashMap<String, String> map) { + if (map.containsKey("from")) { + return Long.valueOf(map.get("from")); + } + return Long.MIN_VALUE; + } + + private long getLatestThreshold(HashMap<String, String> map) { + if (map.containsKey("to")) { + return Long.valueOf(map.get("to")); + } + return Long.MAX_VALUE; + } } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index eb00446dd0e..2483f2497d0 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -10,23 +10,34 @@ import java.nio.file.Files; public class LogReader { - protected static JSONObject readLogs(String logDirectory) throws IOException, JSONException { + long earliestLogThreshold; + long latestLogThreshold; + + public LogReader(long earliestLogThreshold, long latestLogThreshold) { + this.earliestLogThreshold = earliestLogThreshold; + this.latestLogThreshold = latestLogThreshold; + } + + protected JSONObject readLogs(String logDirectory) throws IOException, JSONException { JSONObject json = new JSONObject(); File root = new File(logDirectory); - traverse_folder(root, json); + traverse_folder(root, json, ""); return json; } - private static void traverse_folder(File root, JSONObject json) throws IOException, JSONException { - for(File child : root.listFiles()) { + private void traverse_folder(File root, JSONObject json, String filename) throws IOException, JSONException { + File[] files = root.listFiles(); + for(File child : files) { + File temp = child; JSONObject childJson = new JSONObject(); - if(child.isFile()) { - json.put(child.getName(), DatatypeConverter.printBase64Binary(Files.readAllBytes(child.toPath()))); + long logTime = child.lastModified(); + if(child.isFile() && earliestLogThreshold < logTime && logTime < latestLogThreshold) { + json.put(filename + child.getName(), DatatypeConverter.printBase64Binary(Files.readAllBytes(child.toPath()))); } - else { - json.put(child.getName(), childJson); - traverse_folder(child, childJson); + else if (!child.isFile()){ + traverse_folder(child, json, filename + child.getName() + "-"); } } } + } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index e5302ee43ee..534026f89ac 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -20,8 +20,19 @@ public class LogReaderTest { @Test public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{ String logDirectory = "src/test/resources/logfolder/"; - JSONObject json = LogReader.readLogs(logDirectory); - String expected = "{\"subfolder\":{\"log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"},\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}"; + LogReader logReader = new LogReader(21, Long.MAX_VALUE); + JSONObject json = logReader.readLogs(logDirectory); + String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}"; + String actual = json.toString(); + assertEquals(expected, actual); + } + + @Test + public void testThatLogsOutsideRangeAreExcluded() throws Exception { + String logDirectory = "src/test/resources/logfolder/"; + LogReader logReader = new LogReader(Long.MAX_VALUE, Long.MIN_VALUE); + JSONObject json = logReader.readLogs(logDirectory); + String expected = "{}"; String actual = json.toString(); assertEquals(expected, actual); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index d8e12980472..269d16fd24d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.dispatch.SearchCluster.Group; @@ -21,7 +22,9 @@ public class LoadBalancer { // The implementation here is a simplistic least queries in flight + round-robin load balancer // TODO: consider the options in com.yahoo.vespa.model.content.TuningDispatch - private final static Logger log = Logger.getLogger(LoadBalancer.class.getName()); + private static final Logger log = Logger.getLogger(LoadBalancer.class.getName()); + + private static final CompoundName QUERY_NODE_GROUP_AFFINITY = new CompoundName("dispatch.group.affinity"); private final boolean isInternallyDispatchable; private final List<GroupSchedule> scoreboard; @@ -44,9 +47,9 @@ public class LoadBalancer { /** * Select and allocate the search cluster group which is to be used for the provided query. Callers <b>must</b> call - * {@link #releaseGroup(Group)} symmetrically for each taken allocation. + * {@link #releaseGroup} symmetrically for each taken allocation. * - * @param query + * @param query the query for which this allocation is made * @return The node group to target, or <i>empty</i> if the internal dispatch logic cannot be used */ public Optional<Group> takeGroupForQuery(Query query) { @@ -54,7 +57,16 @@ public class LoadBalancer { return Optional.empty(); } - return allocateNextGroup(); + Integer groupAffinity = query.properties().getInteger(QUERY_NODE_GROUP_AFFINITY); + if (groupAffinity != null) { + Optional<Group> previouslyChosen = allocateFromGroup(groupAffinity); + if (previouslyChosen.isPresent()) { + return previouslyChosen; + } + } + Optional<Group> allocatedGroup = allocateNextGroup(); + allocatedGroup.ifPresent(group -> query.properties().set(QUERY_NODE_GROUP_AFFINITY, group.id())); + return allocatedGroup; } /** @@ -74,6 +86,18 @@ public class LoadBalancer { } } + private Optional<Group> allocateFromGroup(int groupId) { + synchronized (this) { + for (GroupSchedule schedule : scoreboard) { + if (schedule.group.id() == groupId) { + schedule.adjustScore(1); + return Optional.of(schedule.group); + } + } + } + return Optional.empty(); + } + private Optional<Group> allocateNextGroup() { synchronized (this) { GroupSchedule bestSchedule = null; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 2ba991310f5..e94c11e4473 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.yahoo.search.Query; import com.yahoo.search.dispatch.SearchCluster.Group; import com.yahoo.search.dispatch.SearchCluster.Node; import junit.framework.AssertionFailedError; @@ -24,7 +25,7 @@ public class LoadBalancerTest { SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1), null, 1, null); LoadBalancer lb = new LoadBalancer(cluster); - Optional<Group> grp = lb.takeGroupForQuery(null); + Optional<Group> grp = lb.takeGroupForQuery(new Query()); Group group = grp.orElseGet(() -> { throw new AssertionFailedError("Expected a SearchCluster.Group"); }); @@ -38,7 +39,7 @@ public class LoadBalancerTest { SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null); LoadBalancer lb = new LoadBalancer(cluster); - Optional<Group> grp = lb.takeGroupForQuery(null); + Optional<Group> grp = lb.takeGroupForQuery(new Query()); Group group = grp.orElseGet(() -> { throw new AssertionFailedError("Expected a SearchCluster.Group"); }); @@ -52,7 +53,7 @@ public class LoadBalancerTest { SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 2, null); LoadBalancer lb = new LoadBalancer(cluster); - Optional<Group> grp = lb.takeGroupForQuery(null); + Optional<Group> grp = lb.takeGroupForQuery(new Query()); assertThat(grp.isPresent(), is(false)); } @@ -65,7 +66,7 @@ public class LoadBalancerTest { SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2, n3, n4), null, 2, null); LoadBalancer lb = new LoadBalancer(cluster); - Optional<Group> grp = lb.takeGroupForQuery(null); + Optional<Group> grp = lb.takeGroupForQuery(new Query()); assertThat(grp.isPresent(), is(false)); } @@ -77,19 +78,40 @@ public class LoadBalancerTest { LoadBalancer lb = new LoadBalancer(cluster); // get first group - Optional<Group> grp = lb.takeGroupForQuery(null); + Optional<Group> grp = lb.takeGroupForQuery(new Query()); Group group = grp.get(); int id1 = group.id(); // release allocation lb.releaseGroup(group); // get second group - grp = lb.takeGroupForQuery(null); + grp = lb.takeGroupForQuery(new Query()); group = grp.get(); assertThat(group.id(), not(equalTo(id1))); } @Test + public void requreThatLoadBalancerReturnsSameGroupForSameQuery() { + Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); + Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1); + SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null); + LoadBalancer lb = new LoadBalancer(cluster); + + Query q = new Query(); + // get first group + Optional<Group> grp = lb.takeGroupForQuery(q); + Group group = grp.get(); + int id1 = group.id(); + // release allocation + lb.releaseGroup(group); + + // continue with same query + grp = lb.takeGroupForQuery(q); + group = grp.get(); + assertThat(group.id(), equalTo(id1)); + } + + @Test public void requreThatLoadBalancerReturnsGroupWithShortestQueue() { Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1); @@ -97,12 +119,12 @@ public class LoadBalancerTest { LoadBalancer lb = new LoadBalancer(cluster); // get first group - Optional<Group> grp = lb.takeGroupForQuery(null); + Optional<Group> grp = lb.takeGroupForQuery(new Query()); Group group = grp.get(); int id1 = group.id(); // get second group - grp = lb.takeGroupForQuery(null); + grp = lb.takeGroupForQuery(new Query()); group = grp.get(); int id2 = group.id(); assertThat(id2, not(equalTo(id1))); @@ -110,7 +132,7 @@ public class LoadBalancerTest { lb.releaseGroup(group); // get third group - grp = lb.takeGroupForQuery(null); + grp = lb.takeGroupForQuery(new Query()); group = grp.get(); assertThat(group.id(), equalTo(id2)); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index eb10c78f891..5dacaf9b0db 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; -import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -9,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -42,7 +42,7 @@ public interface ConfigServer { Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath); - HttpResponse getLogs(DeploymentId deployment); + Optional<Logs> getLogs(DeploymentId deployment, HashMap<String, String> queryParameters); /** * Set new status on en endpoint in one zone. * diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java new file mode 100644 index 00000000000..223d3e88f13 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java @@ -0,0 +1,16 @@ +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import java.util.Map; + +public class Logs { + + private final Map<String, String> logs; + + public Logs(Map<String, String> logs) { + this.logs = logs; + } + + public Map<String, String> logs() { + return this.logs; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTrustStoreConfigurator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTrustStoreConfigurator.java deleted file mode 100644 index e805332429b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTrustStoreConfigurator.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.filter; - -import com.google.inject.Inject; -import com.yahoo.jdisc.http.ssl.SslTrustStoreConfigurator; -import com.yahoo.jdisc.http.ssl.SslTrustStoreContext; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; -import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.KeyStore; - -/** - * Load trust store with Athenz CA certificates - * - * @author bjorncs - */ -public class AthenzTrustStoreConfigurator implements SslTrustStoreConfigurator { - - private final KeyStore trustStore; - - @Inject - public AthenzTrustStoreConfigurator(AthenzConfig config) { - this.trustStore = createTrustStore(Paths.get(config.athenzCaTrustStore())); - } - - private static KeyStore createTrustStore(Path trustStoreFile) { - return KeyStoreBuilder.withType(KeyStoreType.JKS) - .fromFile(trustStoreFile, "changeit".toCharArray()) - .build(); - } - - @Override - public void configure(SslTrustStoreContext context) { - context.updateTrustStore(trustStore); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index becef782519..2284b82bcfb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -412,10 +412,13 @@ public class DeploymentTrigger { */ public boolean isComplete(Change change, Application application, JobType jobType) { Optional<Deployment> existingDeployment = deploymentFor(application, jobType); - return successOn(application, jobType, Versions.from(change, application, existingDeployment, controller.systemVersion())).isPresent() + return application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::lastSuccess) + .map(job -> change.platform().map(job.platform()::equals).orElse(true) + && change.application().map(job.application()::equals).orElse(true)) + .orElse(false) || jobType.isProduction() && existingDeployment.map(deployment -> ! isUpgrade(change, deployment) && isDowngrade(application.change(), deployment)) - .orElse(false); + .orElse(false); } private static boolean isUpgrade(Change change, Deployment deployment) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index a83c9cdf06f..7f3b2400736 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName; @@ -39,12 +38,6 @@ public class OsUpgrader extends InfrastructureUpgrader { } @Override - protected void maintain() { - if (controller().system() != SystemName.cd) return; // TODO: Enable in all systems - super.maintain(); - } - - @Override protected void upgrade(Version target, SystemApplication application, ZoneId zone) { if (wantedVersion(zone, application, target).equals(target)) { return; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 034db3d487d..154c4e632de 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -49,6 +49,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFact import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; @@ -87,7 +88,9 @@ import java.net.URISyntaxException; import java.security.Principal; import java.time.DayOfWeek; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -168,7 +171,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.getUri().getQuery()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); @@ -346,13 +349,28 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region) { + private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region, String query) { ApplicationId application = ApplicationId.from(tenantName, applicationName, instanceName); ZoneId zone = ZoneId.from(environment, region); DeploymentId deployment = new DeploymentId(application, zone); - return controller.configServer().getLogs(deployment); + HashMap<String, String> queryParameters = getParameters(query); + Optional<Logs> response = controller.configServer().getLogs(deployment, queryParameters); + Slime slime = new Slime(); + Cursor object = slime.setObject(); + if (response.isPresent()) { + response.get().logs().entrySet().stream().forEach(entry -> object.setString(entry.getKey(), entry.getValue())); + } + return new SlimeJsonResponse(slime); } + private HashMap<String, String> getParameters(String query) { + HashMap<String, String> keyValPair = new HashMap<>(); + Arrays.stream(query.split("&")).forEach(pair -> { + String[] splitPair = pair.split("="); + keyValPair.put(splitPair[0], splitPair[1]); + }); + return keyValPair; + } private void toSlime(Cursor object, Application application, HttpRequest request) { object.setString("application", application.id().application().value()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index db38b2a0084..4adea3383c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -147,9 +147,9 @@ class JobControllerApiHandlerHelper { lastPlatformObject.setLong("at", lastVespa.committedAt().toEpochMilli()); long completed = steps.productionJobs().stream().filter(type -> controller.applications().deploymentTrigger().isComplete(Change.of(lastPlatform), application, type)).count(); if (Optional.of(lastPlatform).equals(change.platform())) - lastPlatformObject.setString("deploying", completed + " of " + steps.productionJobs().size() + "complete"); + lastPlatformObject.setString("deploying", completed + " of " + steps.productionJobs().size() + " complete"); else if (completed == steps.productionJobs().size()) - lastPlatformObject.setString("completed", completed + " of " + steps.productionJobs().size() + "complete"); + lastPlatformObject.setString("completed", completed + " of " + steps.productionJobs().size() + " complete"); else if ( ! application.deploymentSpec().canUpgradeAt(controller.clock().instant())) { lastPlatformObject.setString("blocked", application.deploymentSpec().changeBlocker().stream() .filter(blocker -> blocker.blocksVersions()) @@ -157,7 +157,10 @@ class JobControllerApiHandlerHelper { .findAny().map(blocker -> blocker.window().toString()).get()); } else - lastPlatformObject.setString("pending", "Waiting for current deployment to complete"); + lastPlatformObject.setString("pending", + application.changeAt(controller.clock().instant()).isPresent() + ? "Waiting for current deployment to complete" + : "Waiting for upgrade slot"); } private static void lastApplicationToSlime(Cursor lastApplicationObject, Application application, Change change, DeploymentSteps steps, Controller controller) { @@ -254,7 +257,11 @@ class JobControllerApiHandlerHelper { break; for (JobType stepType : steps.toJobs(step)) { if (pendingProduction.containsKey(stepType)) { - pendingObject.setString(shortNameOf(stepType, controller.system()), statusOf(controller, application.id(), stepType, versions)); + Versions jobVersions = Versions.from(application.changeAt(controller.clock().instant()), + application, + Optional.ofNullable(application.deployments().get(stepType.zone(controller.system()))), + controller.systemVersion()); + pendingObject.setString(shortNameOf(stepType, controller.system()), statusOf(controller, application.id(), stepType, jobVersions)); if (++pending == 3) break steps; } @@ -275,8 +282,8 @@ class JobControllerApiHandlerHelper { private static String statusOf(Controller controller, ApplicationId id, JobType type, Versions versions) { return controller.jobController().last(id, type) - .filter(run -> versions.targetsMatch(versions)) - .filter(run -> type == systemTest || versions.sourcesMatchIfPresent(versions)) + .filter(run -> run.versions().targetsMatch(versions)) + .filter(run -> type != stagingTest || run.versions().sourcesMatchIfPresent(versions)) .map(JobControllerApiHandlerHelper::taskStatusOf) .orElse("pending"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index bd65465633e..aea809de365 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; @@ -298,14 +299,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public HttpResponse getLogs(DeploymentId deployment) { - return new HttpResponse(200) { - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write("{\"subfolder\":{\"log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"},\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}".getBytes()); - } - }; - + public Optional<Logs> getLogs(DeploymentId deployment, HashMap<String, String> queryParameters) { + HashMap<String, String> logs = new HashMap<>(); + logs.put("subfolder-log2.log", "VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl"); + logs.put("log1.log", "VGhpcyBpcyBvbmUgbG9nIGZpbGU="); + return Optional.of(new Logs(logs)); } public static class Application { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 045386dd93a..74f7ab7faf2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -119,33 +119,6 @@ public class OsUpgraderTest { .allMatch(node -> node.version().equals(version1))); } - // TODO: Remove once enabled in all systems - @Test - public void os_upgrade_in_main_does_nothing() { - OsUpgrader osUpgrader = osUpgrader( - UpgradePolicy.create() - .upgrade(zone1) - .upgradeInParallel(zone2, zone3) - .upgrade(zone4), - SystemName.main - ); - - // Bootstrap system - tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4, zone5), - singletonList(SystemApplication.zone), - Optional.of(NodeType.host)); - - // New OS is released - CloudName cloud = CloudName.defaultName(); - Version version1 = Version.fromString("7.1"); - tester.controller().upgradeOsIn(cloud, version1); - statusUpdater.maintain(); - - // Nothing happens as main is explicitly disabled - osUpgrader.maintain(); - assertWanted(Version.emptyVersion, SystemApplication.zone, zone1); - } - private List<OsVersionStatus.Node> nodesOn(Version version) { return tester.controller().osVersionStatus().versions().entrySet().stream() .filter(entry -> entry.getKey().version().equals(version)) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 0ea23ae1b78..30c81a0721a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -331,7 +331,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("application1-recursive.json")); // GET logs - tester.assertResponse(request("/application/v4/tenant/tenant2/application//application1/environment/prod/region/corp-us-east-1/instance/default/logs", GET).userIdentity(USER_ID), new File("logs.json")); + tester.assertResponse(request("/application/v4/tenant/tenant2/application//application1/environment/prod/region/corp-us-east-1/instance/default/logs?from=1233&to=3214", GET).userIdentity(USER_ID), new File("logs.json")); // DELETE (cancel) ongoing change tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index f6b33940929..4c8cf0e7784 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -101,7 +101,7 @@ public class JobControllerApiHandlerHelperTest { tester.tester().upgradeSystem(platform); // us-central-1 has started, deployed, and is installing. Deployment is not yet verified. - // us-east-3 is pending the failed staging test, while us-east-3 is pending us-central-1. + // us-east-3 is waiting for the failed staging test and us-central-1, while us-west-1 is waiting only for us-central-1. // Only us-east-3 is verified, on revision1. // staging-test has 4 runs: one success without sources on revision1, one success from revision1 to revision2, // one success from revision2 to revision3 and one failure from revision1 to revision3. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json index 398a62758ee..69fc0f88ea6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json @@ -1,5 +1,4 @@ { - "subfolder": { - "log2.log":"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl"}, + "subfolder-log2.log":"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl", "log1.log":"VGhpcyBpcyBvbmUgbG9nIGZpbGU=" }
\ No newline at end of file diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index abdbf394591..3f34314f0de 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -246,9 +246,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com } public boolean equals(Object other) { - if (!(other instanceof DataType)) return false; - DataType type = (DataType)other; - return (name.equals(type.name) && dataTypeId == type.dataTypeId); + return (other instanceof DataType) && (dataTypeId == ((DataType)other).dataTypeId); } public String toString() { diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 4ae5e6a713c..7678360ea30 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -70,9 +70,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub log.log(LogLevel.DEBUG, "Configuring document manager with " + config.datatype().size() + " data types."); ArrayList<DocumentmanagerConfig.Datatype> failed = new ArrayList<>(); failed.addAll(config.datatype()); - int failCounter = 30; while (!failed.isEmpty()) { - --failCounter; ArrayList<DocumentmanagerConfig.Datatype> tmp = failed; failed = new ArrayList<>(); for (int i = 0; i < tmp.size(); i++) { @@ -82,9 +80,6 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub registerTypeIdMapping(config, manager, thisDataType, id); } catch (IllegalArgumentException e) { failed.add(thisDataType); - if (failCounter < 0) { - throw e; - } } } } diff --git a/document/src/main/java/com/yahoo/document/ReferenceDataType.java b/document/src/main/java/com/yahoo/document/ReferenceDataType.java index 5b5ba256f43..115917c4118 100644 --- a/document/src/main/java/com/yahoo/document/ReferenceDataType.java +++ b/document/src/main/java/com/yahoo/document/ReferenceDataType.java @@ -75,6 +75,7 @@ public class ReferenceDataType extends DataType { "type in ReferenceDataType instance (type is '%s')", this.targetType.getName())); } this.targetType = targetType; + setName(buildTypeName(targetType)); } @Override @@ -98,4 +99,21 @@ public class ReferenceDataType extends DataType { ReferenceFieldValue rhs = (ReferenceFieldValue)value; return rhs.getDataType().equals(this); } + + private int compareTargetType(DataType rhs) { + return (rhs instanceof ReferenceDataType) ? targetType.compareTo(((ReferenceDataType) rhs).targetType) : 0; + } + + @Override + public int compareTo(DataType rhs) { + int cmp = super.compareTo(rhs); + return (cmp != 0) ? cmp : compareTargetType(rhs); + } + + @Override + public boolean equals(Object rhs) { + return super.equals(rhs) + && (rhs instanceof ReferenceDataType) + && targetType.equals(((ReferenceDataType) rhs).targetType); + } } diff --git a/document/src/test/document/documentmanager.replaced_temporary.cfg b/document/src/test/document/documentmanager.replaced_temporary.cfg new file mode 100644 index 00000000000..06a5364cfb1 --- /dev/null +++ b/document/src/test/document/documentmanager.replaced_temporary.cfg @@ -0,0 +1,55 @@ +enablecompression false +datatype[12].id 1191756824 +datatype[12].referencetype[0].target_type_id 886149367 +datatype[13].id -2054976470 +datatype[13].arraytype[0].datatype 5 +datatype[14].id 959075962 +datatype[14].structtype[0].name "ad.header" +datatype[14].structtype[0].version 0 +datatype[14].structtype[0].compresstype "NONE" +datatype[14].structtype[0].compresslevel 0 +datatype[14].structtype[0].compressthreshold 95 +datatype[14].structtype[0].compressminsize 800 +datatype[14].structtype[0].field[286].datatype 1191756824 +datatype[14].structtype[0].field[286].name "campaign_ref" +datatype[14].structtype[0].field[286].detailedtype "" +datatype[15].id -255288561 +datatype[15].structtype[0].name "ad.body" +datatype[15].structtype[0].version 0 +datatype[15].structtype[0].compresstype "NONE" +datatype[15].structtype[0].compresslevel 0 +datatype[15].structtype[0].compressthreshold 95 +datatype[15].structtype[0].compressminsize 800 +datatype[16].id 2987301 +datatype[16].documenttype[0].name "ad" +datatype[16].documenttype[0].version 0 +datatype[16].documenttype[0].inherits[0].name "document" +datatype[16].documenttype[0].inherits[0].version 0 +datatype[16].documenttype[0].headerstruct 959075962 +datatype[16].documenttype[0].bodystruct -255288561 +datatype[16].documenttype[0].fieldsets.[document].fields[722] "campaign_ref" +datatype[57].id 350014056 +datatype[57].structtype[0].name "mystiqueCampaign.header" +datatype[57].structtype[0].version 0 +datatype[57].structtype[0].compresstype "NONE" +datatype[57].structtype[0].compresslevel 0 +datatype[57].structtype[0].compressthreshold 95 +datatype[57].structtype[0].compressminsize 800 +datatype[57].structtype[0].field[0].datatype 4 +datatype[57].structtype[0].field[0].name "campaign_id" +datatype[57].structtype[0].field[0].detailedtype "" +datatype[58].id -524078467 +datatype[58].structtype[0].name "mystiqueCampaign.body" +datatype[58].structtype[0].version 0 +datatype[58].structtype[0].compresstype "NONE" +datatype[58].structtype[0].compresslevel 0 +datatype[58].structtype[0].compressthreshold 95 +datatype[58].structtype[0].compressminsize 800 +datatype[59].id 886149367 +datatype[59].documenttype[0].name "mystiqueCampaign" +datatype[59].documenttype[0].version 0 +datatype[59].documenttype[0].inherits[0].name "document" +datatype[59].documenttype[0].inherits[0].version 0 +datatype[59].documenttype[0].headerstruct 350014056 +datatype[59].documenttype[0].bodystruct -524078467 +datatype[59].documenttype[0].fieldsets.[document].fields[0] "campaign_id" diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java index aa4f5211df7..6acae4f37c6 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java @@ -528,6 +528,16 @@ search annotationsimplicitstruct { } @Test + public void no_temporary_targets_in_references_or_names() { + DocumentTypeManager manager = createConfiguredManager("file:src/test/document/documentmanager.replaced_temporary.cfg"); + DocumentType docType = manager.getDocumentType("ad"); + Field f = docType.getField("campaign_ref"); + assertTrue(f.getDataType() instanceof ReferenceDataType); + assertFalse(((ReferenceDataType)f.getDataType()).getTargetType() instanceof TemporaryStructuredDataType); + assertEquals("Reference<mystiqueCampaign>", f.getDataType().getName()); + } + + @Test public void can_have_reference_type_pointing_to_own_document_type() { DocumentTypeManager manager = createConfiguredManager("file:src/test/document/documentmanager.selfreference.cfg"); diff --git a/document/src/vespa/document/datatype/datatype.cpp b/document/src/vespa/document/datatype/datatype.cpp index aef155999a4..8d2721a4d9b 100644 --- a/document/src/vespa/document/datatype/datatype.cpp +++ b/document/src/vespa/document/datatype/datatype.cpp @@ -159,7 +159,7 @@ DataType::~DataType() = default; bool DataType::operator==(const DataType& other) const { - return _dataTypeId == other._dataTypeId && _name == other._name; + return _dataTypeId == other._dataTypeId; } bool diff --git a/document/src/vespa/document/datatype/referencedatatype.cpp b/document/src/vespa/document/datatype/referencedatatype.cpp index 6792d95909c..7b7c83c7fa6 100644 --- a/document/src/vespa/document/datatype/referencedatatype.cpp +++ b/document/src/vespa/document/datatype/referencedatatype.cpp @@ -41,4 +41,10 @@ void ReferenceDataType::onBuildFieldPath(FieldPath &, vespalib::stringref remain } +bool ReferenceDataType::operator==(const DataType &rhs) const { + return DataType::operator==(rhs) + && rhs.inherits(classId) + && (_targetDocType == static_cast<const ReferenceDataType &>(rhs)._targetDocType); +} + } // document diff --git a/document/src/vespa/document/datatype/referencedatatype.h b/document/src/vespa/document/datatype/referencedatatype.h index d5804d09835..5ca52f3ccb2 100644 --- a/document/src/vespa/document/datatype/referencedatatype.h +++ b/document/src/vespa/document/datatype/referencedatatype.h @@ -24,6 +24,8 @@ public: void print(std::ostream&, bool verbose, const std::string& indent) const override; ReferenceDataType* clone() const override; void onBuildFieldPath(FieldPath & path, vespalib::stringref remainingFieldName) const override; + + bool operator==(const DataType &type) const override; }; } // document diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp index 3ccb08c32be..7c308202e3b 100644 --- a/document/src/vespa/document/datatype/structdatatype.cpp +++ b/document/src/vespa/document/datatype/structdatatype.cpp @@ -40,7 +40,7 @@ StructDataType::StructDataType(vespalib::stringref name, int32_t dataTypeId) _compressionConfig() { } -StructDataType::~StructDataType() { } +StructDataType::~StructDataType() = default; StructDataType* StructDataType::clone() const { diff --git a/document/src/vespa/document/datatype/structdatatype.h b/document/src/vespa/document/datatype/structdatatype.h index 4491ed68e01..42003d3b466 100644 --- a/document/src/vespa/document/datatype/structdatatype.h +++ b/document/src/vespa/document/datatype/structdatatype.h @@ -71,10 +71,10 @@ public: DECLARE_IDENTIFIABLE(StructDataType); private: - typedef vespalib::hash_map<vespalib::string, Field::SP> StringFieldMap; - typedef vespalib::hash_map<int32_t, Field::SP> IntFieldMap; - StringFieldMap _nameFieldMap; - IntFieldMap _idFieldMap; + using StringFieldMap = vespalib::hash_map<vespalib::string, Field::SP>; + using IntFieldMap = vespalib::hash_map<int32_t, Field::SP>; + StringFieldMap _nameFieldMap; + IntFieldMap _idFieldMap; CompressionConfig _compressionConfig; /** @return "" if not conflicting. Error message otherwise. */ diff --git a/document/src/vespa/document/repo/configbuilder.cpp b/document/src/vespa/document/repo/configbuilder.cpp index 45433c2a606..42b37104e04 100644 --- a/document/src/vespa/document/repo/configbuilder.cpp +++ b/document/src/vespa/document/repo/configbuilder.cpp @@ -2,8 +2,8 @@ #include "configbuilder.h" -namespace document { -namespace config_builder { +namespace document::config_builder { + int32_t createFieldId(const vespalib::string &name, int32_t type) { StructDataType dummy("dummy", type); Field f(name, dummy, true); @@ -63,5 +63,4 @@ DocumenttypesConfigBuilderHelper::document(int32_t id, const vespalib::string &n return DocTypeRep(_config.documenttype.back()); } -} // namespace config_builder -} // namespace document +} diff --git a/document/src/vespa/document/repo/configbuilder.h b/document/src/vespa/document/repo/configbuilder.h index 598c72f6358..c389fd3b09e 100644 --- a/document/src/vespa/document/repo/configbuilder.h +++ b/document/src/vespa/document/repo/configbuilder.h @@ -9,8 +9,7 @@ #include <vespa/vespalib/stllike/string.h> #include <cassert> -namespace document { -namespace config_builder { +namespace document::config_builder { class TypeOrId; @@ -143,6 +142,6 @@ public: ::document::DocumenttypesConfigBuilder &config() { return _config; } }; -} // namespace config_builder -} // namespace document + +} diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index 03b7660efbe..a320750e0d5 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -89,7 +89,7 @@ void Repo::inherit(const Repo &parent) { bool Repo::addDataType(const DataType &type) { const DataType *& data_type = _types[type.getId()]; if (data_type) { - if (*data_type == type) { + if ((*data_type == type) && (data_type->getName() == type.getName())) { return false; // Redefinition of identical type is ok. } throw IllegalArgumentException( diff --git a/documentgen-test/etc/complex/music.sd b/documentgen-test/etc/complex/music.sd index 6dbb7b76862..b95edd2b4f3 100644 --- a/documentgen-test/etc/complex/music.sd +++ b/documentgen-test/etc/complex/music.sd @@ -46,12 +46,12 @@ search music { field sw1 type float { indexing { - input weight * 6 + input w1 + input w2 | summary; + input weight_src * 6 + input w1_src + input w2_src | summary; } } field didinteger type array<int> { - indexing: input did | split " " | attribute + indexing: input did | split " " | for_each { to_int } | attribute } rank-profile default { diff --git a/documentgen-test/etc/complex/music2.sd b/documentgen-test/etc/complex/music2.sd index e608225bf38..4cc9db0651e 100644 --- a/documentgen-test/etc/complex/music2.sd +++ b/documentgen-test/etc/complex/music2.sd @@ -51,12 +51,12 @@ search music2 { field sw1 type float { indexing { - input weight * 6 + input w1 + input w2 | summary; + input weight_src * 6 + input w1_src + input w2_src | summary; } } field didinteger type array<int> { - indexing: input did | split " " | attribute + indexing: input did | split " " | for_each { to_int } | attribute } rank-profile default { diff --git a/documentgen-test/etc/complex/video.sd b/documentgen-test/etc/complex/video.sd index fc7f58298c1..0749daa01aa 100644 --- a/documentgen-test/etc/complex/video.sd +++ b/documentgen-test/etc/complex/video.sd @@ -32,7 +32,7 @@ search video { field sw1 type float { indexing { - input weight * 6 + input w1 + input w2 | summary; + input weight_src * 6 + input w1_src + input w2_src | summary; } } diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp index f2864d1dd58..e028afe5deb 100644 --- a/fnet/src/vespa/fnet/connection.cpp +++ b/fnet/src/vespa/fnet/connection.cpp @@ -225,6 +225,7 @@ FNET_Connection::handshake() case vespalib::CryptoSocket::HandshakeResult::DONE: { EnableReadEvent(true); EnableWriteEvent(writePendingAfterConnect()); + _flags._framed = (_socket->min_read_buffer_size() > 1); size_t chunk_size = std::max(size_t(FNET_READ_SIZE), _socket->min_read_buffer_size()); ssize_t res = 0; do { // drain input pipeline @@ -287,7 +288,7 @@ FNET_Connection::Read() _input.FreeToData((uint32_t)res); broken = !handle_packets(); _input.resetIfEmpty(); - if (broken || (_input.GetFreeLen() > 0) || (readCnt >= FNET_READ_REDO)) { + if (broken || ((_input.GetFreeLen() > 0) && !_flags._framed) || (readCnt >= FNET_READ_REDO)) { goto done_read; } _input.EnsureFree(chunk_size); @@ -302,7 +303,6 @@ done_read: _input.EnsureFree(chunk_size); res = _socket->drain(_input.GetFree(), _input.GetFreeLen()); my_errno = errno; - readCnt++; if (res > 0) { _input.FreeToData((uint32_t)res); broken = !handle_packets(); @@ -340,6 +340,7 @@ done_read: bool FNET_Connection::Write() { + size_t chunk_size = std::max(size_t(FNET_WRITE_SIZE), _socket->min_read_buffer_size()); uint32_t my_write_work = 0; int writeCnt = 0; // write count bool broken = false; // is this conn broken ? @@ -353,7 +354,7 @@ FNET_Connection::Write() // fill output buffer - while (_output.GetDataLen() < FNET_WRITE_SIZE) { + while (_output.GetDataLen() < chunk_size) { if (_myQueue.IsEmpty_NoLock()) break; diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h index 8e275d68b18..44ad3fea97d 100644 --- a/fnet/src/vespa/fnet/connection.h +++ b/fnet/src/vespa/fnet/connection.h @@ -70,12 +70,14 @@ private: _gotheader(false), _inCallback(false), _callbackWait(false), - _discarding(false) + _discarding(false), + _framed(false) { } bool _gotheader; bool _inCallback; bool _callbackWait; bool _discarding; + bool _framed; }; struct ResolveHandler : public vespalib::AsyncResolver::ResultHandler { FNET_Connection *connection; diff --git a/fnet/src/vespa/fnet/databuffer.cpp b/fnet/src/vespa/fnet/databuffer.cpp index 3b2e7759c99..74a8bc4e12c 100644 --- a/fnet/src/vespa/fnet/databuffer.cpp +++ b/fnet/src/vespa/fnet/databuffer.cpp @@ -13,7 +13,6 @@ FNET_DataBuffer::FNET_DataBuffer(uint32_t len) if (len > 0) { Alloc::alloc(len).swap(_ownedBuf); - memset(_ownedBuf.get(), 0x55, len); _bufstart = static_cast<char *>(_ownedBuf.get()); assert(_bufstart != nullptr); } else { // len == 0 @@ -70,7 +69,6 @@ FNET_DataBuffer::Shrink(uint32_t newsize) } Alloc newBuf(Alloc::alloc(newsize)); - memset(newBuf.get(), 0x55, newsize); memcpy(newBuf.get(), _datapt, GetDataLen()); _ownedBuf.swap(newBuf); _bufstart = static_cast<char *>(_ownedBuf.get()); @@ -95,7 +93,6 @@ FNET_DataBuffer::Pack(uint32_t needbytes) bufsize *= 2; Alloc newBuf(Alloc::alloc(bufsize)); - memset(newBuf.get(), 0x55, bufsize); memcpy(newBuf.get(), _datapt, GetDataLen()); _ownedBuf.swap(newBuf); _bufstart = static_cast<char *>(_ownedBuf.get()); diff --git a/fnet/src/vespa/fnet/transport_thread.cpp b/fnet/src/vespa/fnet/transport_thread.cpp index b0388bdc140..2c0d00b22f3 100644 --- a/fnet/src/vespa/fnet/transport_thread.cpp +++ b/fnet/src/vespa/fnet/transport_thread.cpp @@ -151,6 +151,32 @@ FNET_TransportThread::DiscardEvent(FNET_ControlPacket *cpacket, } +void +FNET_TransportThread::handle_add_cmd(FNET_IOComponent *ioc) +{ + if (ioc->handle_add_event()) { + AddComponent(ioc); + ioc->_flags._ioc_added = true; + ioc->attach_selector(_selector); + } else { + ioc->Close(); + AddDeleteComponent(ioc); + } +} + + +void +FNET_TransportThread::handle_close_cmd(FNET_IOComponent *ioc) +{ + if (ioc->_flags._ioc_added) { + RemoveComponent(ioc); + ioc->SubRef(); + } + ioc->Close(); + AddDeleteComponent(ioc); +} + + extern "C" { static void pipehandler(int) @@ -423,14 +449,7 @@ FNET_TransportThread::handle_wakeup() switch (packet->GetCommand()) { case FNET_ControlPacket::FNET_CMD_IOC_ADD: - if (context._value.IOC->handle_add_event()) { - AddComponent(context._value.IOC); - context._value.IOC->_flags._ioc_added = true; - context._value.IOC->attach_selector(_selector); - } else { - context._value.IOC->Close(); - AddDeleteComponent(context._value.IOC); - } + handle_add_cmd(context._value.IOC); break; case FNET_ControlPacket::FNET_CMD_IOC_ENABLE_READ: context._value.IOC->EnableReadEvent(true); @@ -442,19 +461,18 @@ FNET_TransportThread::handle_wakeup() break; case FNET_ControlPacket::FNET_CMD_IOC_ENABLE_WRITE: context._value.IOC->EnableWriteEvent(true); - context._value.IOC->SubRef(); + if (context._value.IOC->HandleWriteEvent()) { + context._value.IOC->SubRef(); + } else { + handle_close_cmd(context._value.IOC); + } break; case FNET_ControlPacket::FNET_CMD_IOC_DISABLE_WRITE: context._value.IOC->EnableWriteEvent(false); context._value.IOC->SubRef(); break; case FNET_ControlPacket::FNET_CMD_IOC_CLOSE: - if (context._value.IOC->_flags._ioc_added) { - RemoveComponent(context._value.IOC); - context._value.IOC->SubRef(); - } - context._value.IOC->Close(); - AddDeleteComponent(context._value.IOC); + handle_close_cmd(context._value.IOC); break; } } diff --git a/fnet/src/vespa/fnet/transport_thread.h b/fnet/src/vespa/fnet/transport_thread.h index 1b8d1fa4eeb..408d20619d2 100644 --- a/fnet/src/vespa/fnet/transport_thread.h +++ b/fnet/src/vespa/fnet/transport_thread.h @@ -143,6 +143,9 @@ private: FNET_Config *GetConfig() { return &_config; } + void handle_add_cmd(FNET_IOComponent *ioc); + void handle_close_cmd(FNET_IOComponent *ioc); + public: /** * Construct a transport object. To activate your newly created diff --git a/jdisc_http_service/pom.xml b/jdisc_http_service/pom.xml index f41994c4916..879036db355 100644 --- a/jdisc_http_service/pom.xml +++ b/jdisc_http_service/pom.xml @@ -16,6 +16,7 @@ <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> + <!-- PROVIDED SCOPE --> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk15on</artifactId> @@ -33,6 +34,56 @@ <classifier>no_aop</classifier> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_jetty</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>defaults</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-accesslogging</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <!-- TEST SCOPE --> + <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> <scope>test</scope> @@ -53,12 +104,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_jetty</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> @@ -80,58 +125,10 @@ </exclusions> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-lib</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>defaults</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_core</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>annotations</artifactId> - <version>${project.version}</version> - <scope>provided</scope> + <groupId>org.springframework</groupId> + <artifactId>spring-test</artifactId> + <scope>test</scope> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>component</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-accesslogging</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.jetbrains</groupId> - <artifactId>annotations</artifactId> - <version>13.0</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-test</artifactId> - <scope>test</scope> - </dependency> </dependencies> <build> <plugins> diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java index 8a829d33c1b..f9892759fbd 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java @@ -2,18 +2,9 @@ package com.yahoo.jdisc.http.server.jetty; import com.google.inject.Inject; -import com.yahoo.config.InnerNode; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ConnectorConfig.Ssl; -import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ExcludeCipherSuite; -import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ExcludeProtocol; -import com.yahoo.jdisc.http.ConnectorConfig.Ssl.IncludeCipherSuite; -import com.yahoo.jdisc.http.ConnectorConfig.Ssl.IncludeProtocol; -import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreContext; -import com.yahoo.jdisc.http.ssl.DefaultSslTrustStoreContext; -import com.yahoo.jdisc.http.ssl.SslKeyStoreConfigurator; -import com.yahoo.jdisc.http.ssl.SslTrustStoreConfigurator; +import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; @@ -24,10 +15,6 @@ import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.util.ssl.SslContextFactory; import java.nio.channels.ServerSocketChannel; -import java.util.Arrays; -import java.util.List; -import java.util.function.BiConsumer; -import java.util.function.Function; /** * @author Einar M R Rosenvinge @@ -36,16 +23,13 @@ import java.util.function.Function; public class ConnectorFactory { private final ConnectorConfig connectorConfig; - private final SslKeyStoreConfigurator sslKeyStoreConfigurator; - private final SslTrustStoreConfigurator sslTrustStoreConfigurator; + private final SslContextFactoryProvider sslContextFactoryProvider; @Inject public ConnectorFactory(ConnectorConfig connectorConfig, - SslKeyStoreConfigurator sslKeyStoreConfigurator, - SslTrustStoreConfigurator sslTrustStoreConfigurator) { + SslContextFactoryProvider sslContextFactoryProvider) { this.connectorConfig = connectorConfig; - this.sslKeyStoreConfigurator = sslKeyStoreConfigurator; - this.sslTrustStoreConfigurator = sslTrustStoreConfigurator; + this.sslContextFactoryProvider = sslContextFactoryProvider; } public ConnectorConfig getConnectorConfig() { @@ -87,55 +71,8 @@ public class ConnectorFactory { } private SslConnectionFactory newSslConnectionFactory() { - Ssl sslConfig = connectorConfig.ssl(); - - SslContextFactory factory = new JDiscSslContextFactory(); - - sslKeyStoreConfigurator.configure(new DefaultSslKeyStoreContext(factory)); - sslTrustStoreConfigurator.configure(new DefaultSslTrustStoreContext(factory)); - - switch (sslConfig.clientAuth()) { - case NEED_AUTH: - factory.setNeedClientAuth(true); - break; - case WANT_AUTH: - factory.setWantClientAuth(true); - break; - } - - if (!sslConfig.prng().isEmpty()) { - factory.setSecureRandomAlgorithm(sslConfig.prng()); - } - - // NOTE: ^TLS_RSA_.*$ ciphers are disabled by default in Jetty 9.4.12+ (https://github.com/eclipse/jetty.project/issues/2807) - // JDisc will allow these ciphers by default to support older clients (e.g. Java 8u60 and curl 7.29.0) - String[] excludedCiphersWithoutTlsRsaExclusion = Arrays.stream(factory.getExcludeCipherSuites()) - .filter(cipher -> !cipher.equals("^TLS_RSA_.*$")) - .toArray(String[]::new); - factory.setExcludeCipherSuites(excludedCiphersWithoutTlsRsaExclusion); - - setStringArrayParameter( - factory, sslConfig.excludeProtocol(), ExcludeProtocol::name, SslContextFactory::setExcludeProtocols); - setStringArrayParameter( - factory, sslConfig.includeProtocol(), IncludeProtocol::name, SslContextFactory::setIncludeProtocols); - setStringArrayParameter( - factory, sslConfig.excludeCipherSuite(), ExcludeCipherSuite::name, SslContextFactory::setExcludeCipherSuites); - setStringArrayParameter( - factory, sslConfig.includeCipherSuite(), IncludeCipherSuite::name, SslContextFactory::setIncludeCipherSuites); - - factory.setKeyManagerFactoryAlgorithm(sslConfig.sslKeyManagerFactoryAlgorithm()); - factory.setProtocol(sslConfig.protocol()); + SslContextFactory factory = sslContextFactoryProvider.getInstance(connectorConfig.name(), connectorConfig.listenPort()); return new SslConnectionFactory(factory, HttpVersion.HTTP_1_1.asString()); } - private static <T extends InnerNode> void setStringArrayParameter(SslContextFactory sslContextFactory, - List<T> configValues, - Function<T, String> nameProperty, - BiConsumer<SslContextFactory, String[]> setter) { - if (!configValues.isEmpty()) { - String[] nameArray = configValues.stream().map(nameProperty).toArray(String[]::new); - setter.accept(sslContextFactory, nameArray); - } - } - } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java index 53f330bbc7e..4b01a475842 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java @@ -5,7 +5,6 @@ import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Metric.Context; import com.yahoo.jdisc.http.server.jetty.JettyHttpServer.Metrics; -import org.jetbrains.annotations.Nullable; import java.util.concurrent.atomic.AtomicBoolean; @@ -16,7 +15,7 @@ import java.util.concurrent.atomic.AtomicBoolean; */ public class MetricReporter { private final Metric metric; - private final @Nullable Context context; + private final Context context; private final long requestStartTime; @@ -24,7 +23,7 @@ public class MetricReporter { private final AtomicBoolean firstSetOfTimeToFirstByte = new AtomicBoolean(true); - public MetricReporter(Metric metric, @Nullable Context context, long requestStartTime) { + public MetricReporter(Metric metric, Context context, long requestStartTime) { this.metric = metric; this.context = context; this.requestStartTime = requestStartTime; diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreConfigurator.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreConfigurator.java deleted file mode 100644 index 1cf8997b465..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreConfigurator.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -import com.google.inject.Inject; -import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ssl.pem.PemSslKeyStore; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.KeyStore; -import java.util.logging.Logger; - -/** - * @author bjorncs - */ -public class DefaultSslKeyStoreConfigurator implements SslKeyStoreConfigurator { - - private static final Logger log = Logger.getLogger(DefaultSslKeyStoreConfigurator.class.getName()); - - @SuppressWarnings("deprecation") - private final com.yahoo.jdisc.http.SecretStore secretStore; - private final ConnectorConfig.Ssl config; - - @Inject - @SuppressWarnings("deprecation") - public DefaultSslKeyStoreConfigurator(ConnectorConfig config, com.yahoo.jdisc.http.SecretStore secretStore) { - validateConfig(config.ssl()); - this.secretStore = secretStore; - this.config = config.ssl(); - } - - private static void validateConfig(ConnectorConfig.Ssl config) { - if (!config.enabled()) return; - switch (config.keyStoreType()) { - case JKS: - validateJksConfig(config); - break; - case PEM: - validatePemConfig(config); - break; - } - } - - @Override - public void configure(SslKeyStoreContext context) { - if (!config.enabled()) return; - switch (config.keyStoreType()) { - case JKS: - context.updateKeyStore(config.keyStorePath(), "JKS", secretStore.getSecret(config.keyDbKey())); - break; - case PEM: - context.updateKeyStore(createPemKeyStore(config.pemKeyStore())); - break; - } - } - - private static void validateJksConfig(ConnectorConfig.Ssl ssl) { - if (!ssl.pemKeyStore().keyPath().isEmpty() || ! ssl.pemKeyStore().certificatePath().isEmpty()) { - throw new IllegalArgumentException("pemKeyStore attributes can not be set when keyStoreType is JKS."); - } - if (ssl.keyDbKey().isEmpty()) { - throw new IllegalArgumentException("Missing password for JKS keystore"); - } - } - - private static void validatePemConfig(ConnectorConfig.Ssl ssl) { - if (! ssl.keyStorePath().isEmpty()) { - throw new IllegalArgumentException("keyStorePath can not be set when keyStoreType is PEM"); - } - if (!ssl.keyDbKey().isEmpty()) { - // TODO Make an error once there are separate passwords for truststore and keystore - log.warning("Encrypted PEM key stores are not supported. Password is only applied to truststore"); - } - if (ssl.pemKeyStore().certificatePath().isEmpty()) { - throw new IllegalArgumentException("Missing certificate path."); - } - if (ssl.pemKeyStore().keyPath().isEmpty()) { - throw new IllegalArgumentException("Missing key path."); - } - } - - private static KeyStore createPemKeyStore(ConnectorConfig.Ssl.PemKeyStore pemKeyStore) { - try { - Path certificatePath = Paths.get(pemKeyStore.certificatePath()); - Path keyPath = Paths.get(pemKeyStore.keyPath()); - return new PemSslKeyStore(certificatePath, keyPath).loadJavaKeyStore(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } catch (Exception e) { - throw new RuntimeException("Failed setting up key store for " + pemKeyStore.keyPath() + ", " + pemKeyStore.certificatePath(), e); - } - } - -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreContext.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreContext.java deleted file mode 100644 index 44a9c606576..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreContext.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -import org.eclipse.jetty.util.ssl.SslContextFactory; - -import java.security.KeyStore; -import java.util.function.Consumer; - -/** - * @author bjorncs - */ -public class DefaultSslKeyStoreContext implements SslKeyStoreContext { - - private final SslContextFactory sslContextFactory; - - public DefaultSslKeyStoreContext(SslContextFactory sslContextFactory) { - this.sslContextFactory = sslContextFactory; - } - - @Override - public void updateKeyStore(KeyStore keyStore) { - updateKeyStore(keyStore, null); - } - - @Override - public void updateKeyStore(KeyStore keyStore, String password) { - updateKeyStore(sslContextFactory -> { - sslContextFactory.setKeyStore(keyStore); - if (password != null) { - sslContextFactory.setKeyStorePassword(password); - } - }); - } - - @Override - public void updateKeyStore(String keyStorePath, String keyStoreType, String keyStorePassword) { - updateKeyStore(sslContextFactory -> { - sslContextFactory.setKeyStorePath(keyStorePath); - sslContextFactory.setKeyStoreType(keyStoreType); - sslContextFactory.setKeyStorePassword(keyStorePassword); - }); - } - - private void updateKeyStore(Consumer<SslContextFactory> reloader) { - try { - sslContextFactory.reload(reloader); - } catch (Exception e) { - throw new RuntimeException("Could not update keystore: " + e.getMessage(), e); - } - } -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslTrustStoreConfigurator.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslTrustStoreConfigurator.java deleted file mode 100644 index 5a8c399e6ba..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslTrustStoreConfigurator.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -import com.google.inject.Inject; -import com.yahoo.jdisc.http.ConnectorConfig; - -/** - * @author bjorncs - */ -public class DefaultSslTrustStoreConfigurator implements SslTrustStoreConfigurator { - - @SuppressWarnings("deprecation") - private final com.yahoo.jdisc.http.SecretStore secretStore; - private final ConnectorConfig.Ssl config; - - @Inject - @SuppressWarnings("deprecation") - public DefaultSslTrustStoreConfigurator(ConnectorConfig config, com.yahoo.jdisc.http.SecretStore secretStore) { - validateConfig(config.ssl()); - this.secretStore = secretStore; - this.config = config.ssl(); - } - - @Override - public void configure(SslTrustStoreContext context) { - if (!config.enabled()) return; - String keyDbPassword = config.keyDbKey(); - if (!config.trustStorePath().isEmpty()) { - String password = config.useTrustStorePassword() ? secretStore.getSecret(keyDbPassword) : null; - context.updateTrustStore(config.trustStorePath(), config.trustStoreType().toString(), password); - } - } - - private static void validateConfig(ConnectorConfig.Ssl config) { - if (!config.enabled()) return; - if (!config.trustStorePath().isEmpty() && config.useTrustStorePassword() && config.keyDbKey().isEmpty()) { - throw new IllegalArgumentException("Missing password for JKS truststore"); - } - } - -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslTrustStoreContext.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslTrustStoreContext.java deleted file mode 100644 index c2d91cca3ea..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslTrustStoreContext.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -import org.eclipse.jetty.util.ssl.SslContextFactory; - -import java.security.KeyStore; -import java.util.function.Consumer; - -/** - * @author bjorncs - */ -public class DefaultSslTrustStoreContext implements SslTrustStoreContext { - - private final SslContextFactory sslContextFactory; - - public DefaultSslTrustStoreContext(SslContextFactory sslContextFactory) { - this.sslContextFactory = sslContextFactory; - } - - @Override - public void updateTrustStore(KeyStore trustStore) { - updateTrustStore(trustStore, null); - } - - @Override - public void updateTrustStore(KeyStore trustStore, String password) { - updateTrustStore(sslContextFactory -> { - sslContextFactory.setTrustStore(trustStore); - if (password != null) { - sslContextFactory.setTrustStorePassword(password); - } - }); - } - - @Override - public void updateTrustStore(String trustStorePath, String trustStoreType, String trustStorePassword) { - updateTrustStore(sslContextFactory -> { - sslContextFactory.setTrustStorePath(trustStorePath); - sslContextFactory.setTrustStoreType(trustStoreType); - if (trustStorePassword != null) { - sslContextFactory.setTrustStorePassword(trustStorePassword); - } - }); - } - - private void updateTrustStore(Consumer<SslContextFactory> reloader) { - try { - sslContextFactory.reload(reloader); - } catch (Exception e) { - throw new RuntimeException("Could not update truststore: " + e.getMessage(), e); - } - } - -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java new file mode 100644 index 00000000000..37916fd5734 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java @@ -0,0 +1,20 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl; + +import org.eclipse.jetty.util.ssl.SslContextFactory; + +/** + * A provider that is used to configure SSL connectors in JDisc + * + * @author bjorncs + */ +public interface SslContextFactoryProvider { + + /** + * This method is called once for each SSL connector. + * + * @return returns an instance of {@link SslContextFactory} for a given JDisc http server + */ + SslContextFactory getInstance(String containerId, int port); + +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreConfigurator.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreConfigurator.java deleted file mode 100644 index 619f4a636ed..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreConfigurator.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -/** - * An interface for an component that can configure an {@link SslKeyStoreContext}. The implementor can assume that - * the {@link SslKeyStoreContext} instance is thread-safe and be updated at any time - * during and after the call to{@link #configure(SslKeyStoreContext)}. - * Modifying the {@link SslKeyStoreContext} instance will trigger a hot reload of the keystore in JDisc. - * - * @author bjorncs - */ -public interface SslKeyStoreConfigurator { - void configure(SslKeyStoreContext context); -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreContext.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreContext.java deleted file mode 100644 index 2a25f6d78b5..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreContext.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -import java.security.KeyStore; - -/** - * An interface to update the keystore in JDisc. Any update will trigger a hot reload and new connections will - * immediately see the new certificate chain. - * - * @author bjorncs - */ -public interface SslKeyStoreContext { - void updateKeyStore(KeyStore keyStore); - void updateKeyStore(KeyStore keyStore, String password); - void updateKeyStore(String keyStorePath, String keyStoreType, String keyStorePassword); -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslTrustStoreConfigurator.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslTrustStoreConfigurator.java deleted file mode 100644 index de1119a5275..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslTrustStoreConfigurator.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -/** - * An interface for an component that can configure an {@link SslTrustStoreContext}. The implementor can assume that - * the {@link SslTrustStoreContext} instance is thread-safe and be updated at any time - * during and after the call to{@link #configure(SslTrustStoreContext)}. - * Modifying the {@link SslKeyStoreContext} instance will trigger a hot reload of the truststore in JDisc. - * - * @author bjorncs - */ -public interface SslTrustStoreConfigurator { - void configure(SslTrustStoreContext context); -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslTrustStoreContext.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslTrustStoreContext.java deleted file mode 100644 index fc8cf397b24..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslTrustStoreContext.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -import java.security.KeyStore; - -/** - * An interface to update the truststore in JDisc. Any update will trigger a hot reload and new connections will - * authenticated using the update truststore. - * - * @author bjorncs - */ -public interface SslTrustStoreContext { - void updateTrustStore(KeyStore trustStore); - void updateTrustStore(KeyStore trustStore, String password); - void updateTrustStore(String trustStorePath, String trustStoreType, String trustStorePassword); -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java new file mode 100644 index 00000000000..fa31f58dfc0 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java @@ -0,0 +1,103 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.security.KeyStoreBuilder; +import com.yahoo.security.KeyStoreType; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateUtils; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.Arrays; +import java.util.List; + +/** + * JDisc's default implementation of {@link SslContextFactoryProvider} that uses the {@link ConnectorConfig} to construct a {@link SslContextFactory}. + * + * @author bjorncs + */ +public class DefaultSslContextFactoryProvider implements SslContextFactoryProvider { + + private final ConnectorConfig connectorConfig; + + public DefaultSslContextFactoryProvider(ConnectorConfig connectorConfig) { + validateConfig(connectorConfig.ssl()); + this.connectorConfig = connectorConfig; + } + + @Override + public SslContextFactory getInstance(String containerId, int port) { + ConnectorConfig.Ssl sslConfig = connectorConfig.ssl(); + if (!sslConfig.enabled()) throw new IllegalStateException(); + SslContextFactory factory = new JDiscSslContextFactory(); + + switch (sslConfig.clientAuth()) { + case NEED_AUTH: + factory.setNeedClientAuth(true); + break; + case WANT_AUTH: + factory.setWantClientAuth(true); + break; + } + + // NOTE: All ciphers matching ^TLS_RSA_.*$ are disabled by default in Jetty 9.4.12+ (https://github.com/eclipse/jetty.project/issues/2807) + // JDisc will allow these ciphers by default to support older clients (e.g. Java 8u60 and curl 7.29.0) + // Removing the exclusion will allow for the TLS_RSA variants that are not covered by other exclusions + String[] excludedCiphersWithoutTlsRsaExclusion = Arrays.stream(factory.getExcludeCipherSuites()) + .filter(cipher -> !cipher.equals("^TLS_RSA_.*$")) + .toArray(String[]::new); + factory.setExcludeCipherSuites(excludedCiphersWithoutTlsRsaExclusion); + + // Check if using new ssl syntax from services.xml + factory.setKeyStore(createKeystore(sslConfig)); + factory.setKeyStorePassword(""); + if (!sslConfig.caCertificateFile().isEmpty()) { + factory.setTrustStore(createTruststore(sslConfig)); + } + factory.setProtocol("TLS"); + return factory; + } + + private static void validateConfig(ConnectorConfig.Ssl config) { + if (!config.enabled()) return; + if (config.certificateFile().isEmpty()) { + throw new IllegalArgumentException("Missing certificate file."); + } + if (config.privateKeyFile().isEmpty()) { + throw new IllegalArgumentException("Missing private key file."); + } + + } + + private static KeyStore createTruststore(ConnectorConfig.Ssl sslConfig) { + List<X509Certificate> caCertificates = X509CertificateUtils.certificateListFromPem(readToString(sslConfig.caCertificateFile())); + KeyStoreBuilder truststoreBuilder = KeyStoreBuilder.withType(KeyStoreType.JKS); + for (int i = 0; i < caCertificates.size(); i++) { + truststoreBuilder.withCertificateEntry("entry-" + i, caCertificates.get(i)); + } + return truststoreBuilder.build(); + } + + private static KeyStore createKeystore(ConnectorConfig.Ssl sslConfig) { + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(readToString(sslConfig.privateKeyFile())); + List<X509Certificate> certificates = X509CertificateUtils.certificateListFromPem(readToString(sslConfig.certificateFile())); + return KeyStoreBuilder.withType(KeyStoreType.JKS).withKeyEntry("default", privateKey, certificates).build(); + } + + private static String readToString(String filename) { + try { + return new String(Files.readAllBytes(Paths.get(filename))); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscSslContextFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java index 81a6a0c8048..90d67996402 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscSslContextFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.server.jetty; +package com.yahoo.jdisc.http.ssl.impl; import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.security.CertificateUtils; diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/LegacySslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/LegacySslContextFactoryProvider.java new file mode 100644 index 00000000000..281f80c3aeb --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/LegacySslContextFactoryProvider.java @@ -0,0 +1,164 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import com.yahoo.config.InnerNode; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.pem.PemSslKeyStore; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.util.Arrays; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.logging.Logger; + +/** + * A implementation of {@link SslContextFactoryProvider} to be injected into non-ssl connectors or connectors using legacy ssl config + * + * @author bjorncs + */ +// TODO Vespa 7: Remove legacy ssl config +public class LegacySslContextFactoryProvider implements SslContextFactoryProvider { + private static final Logger log = Logger.getLogger(LegacySslContextFactoryProvider.class.getName()); + + private final ConnectorConfig connectorConfig; + @SuppressWarnings("deprecation") + private final com.yahoo.jdisc.http.SecretStore secretStore; + + public LegacySslContextFactoryProvider(ConnectorConfig connectorConfig, + @SuppressWarnings("deprecation") com.yahoo.jdisc.http.SecretStore secretStore) { + validateConfig(connectorConfig.ssl()); + this.connectorConfig = connectorConfig; + this.secretStore = secretStore; + } + + @Override + public SslContextFactory getInstance(String containerId, int port) { + ConnectorConfig.Ssl sslConfig = connectorConfig.ssl(); + if (!sslConfig.enabled()) throw new IllegalStateException(); + SslContextFactory factory = new JDiscSslContextFactory(); + + switch (sslConfig.clientAuth()) { + case NEED_AUTH: + factory.setNeedClientAuth(true); + break; + case WANT_AUTH: + factory.setWantClientAuth(true); + break; + } + + // NOTE: All ciphers matching ^TLS_RSA_.*$ are disabled by default in Jetty 9.4.12+ (https://github.com/eclipse/jetty.project/issues/2807) + // JDisc will allow these ciphers by default to support older clients (e.g. Java 8u60 and curl 7.29.0) + // Removing the exclusion will allow for the TLS_RSA variants that are not covered by other exclusions + String[] excludedCiphersWithoutTlsRsaExclusion = Arrays.stream(factory.getExcludeCipherSuites()) + .filter(cipher -> !cipher.equals("^TLS_RSA_.*$")) + .toArray(String[]::new); + factory.setExcludeCipherSuites(excludedCiphersWithoutTlsRsaExclusion); + + switch (sslConfig.keyStoreType()) { + case JKS: + factory.setKeyStorePath(sslConfig.keyStorePath()); + factory.setKeyStoreType("JKS"); + factory.setKeyStorePassword(secretStore.getSecret(sslConfig.keyDbKey())); + break; + case PEM: + factory.setKeyStorePath(sslConfig.keyStorePath()); + factory.setKeyStore(createPemKeyStore(sslConfig.pemKeyStore())); + break; + } + + if (!sslConfig.trustStorePath().isEmpty()) { + factory.setTrustStorePath(sslConfig.trustStorePath()); + factory.setTrustStoreType(sslConfig.trustStoreType().toString()); + if (sslConfig.useTrustStorePassword()) { + factory.setTrustStorePassword(secretStore.getSecret(sslConfig.keyDbKey())); + } + } + + if (!sslConfig.prng().isEmpty()) { + factory.setSecureRandomAlgorithm(sslConfig.prng()); + } + + setStringArrayParameter( + factory, sslConfig.excludeProtocol(), ConnectorConfig.Ssl.ExcludeProtocol::name, SslContextFactory::setExcludeProtocols); + setStringArrayParameter( + factory, sslConfig.includeProtocol(), ConnectorConfig.Ssl.IncludeProtocol::name, SslContextFactory::setIncludeProtocols); + setStringArrayParameter( + factory, sslConfig.excludeCipherSuite(), ConnectorConfig.Ssl.ExcludeCipherSuite::name, SslContextFactory::setExcludeCipherSuites); + setStringArrayParameter( + factory, sslConfig.includeCipherSuite(), ConnectorConfig.Ssl.IncludeCipherSuite::name, SslContextFactory::setIncludeCipherSuites); + + factory.setKeyManagerFactoryAlgorithm(sslConfig.sslKeyManagerFactoryAlgorithm()); + factory.setProtocol(sslConfig.protocol()); + + return factory; + } + + private static void validateConfig(ConnectorConfig.Ssl config) { + if (!config.enabled()) return; + switch (config.keyStoreType()) { + case JKS: + validateJksConfig(config); + break; + case PEM: + validatePemConfig(config); + break; + } + if (!config.trustStorePath().isEmpty() && config.useTrustStorePassword() && config.keyDbKey().isEmpty()) { + throw new IllegalArgumentException("Missing password for JKS truststore"); + } + } + + private static void validateJksConfig(ConnectorConfig.Ssl ssl) { + if (!ssl.pemKeyStore().keyPath().isEmpty() || ! ssl.pemKeyStore().certificatePath().isEmpty()) { + throw new IllegalArgumentException("pemKeyStore attributes can not be set when keyStoreType is JKS."); + } + if (ssl.keyDbKey().isEmpty()) { + throw new IllegalArgumentException("Missing password for JKS keystore"); + } + } + + private static void validatePemConfig(ConnectorConfig.Ssl ssl) { + if (! ssl.keyStorePath().isEmpty()) { + throw new IllegalArgumentException("keyStorePath can not be set when keyStoreType is PEM"); + } + if (!ssl.keyDbKey().isEmpty()) { + log.warning("Encrypted PEM key stores are not supported. Password is only applied to truststore"); + } + if (ssl.pemKeyStore().certificatePath().isEmpty()) { + throw new IllegalArgumentException("Missing certificate path."); + } + if (ssl.pemKeyStore().keyPath().isEmpty()) { + throw new IllegalArgumentException("Missing key path."); + } + } + + private static KeyStore createPemKeyStore(ConnectorConfig.Ssl.PemKeyStore pemKeyStore) { + try { + Path certificatePath = Paths.get(pemKeyStore.certificatePath()); + Path keyPath = Paths.get(pemKeyStore.keyPath()); + return new PemSslKeyStore(certificatePath, keyPath).loadJavaKeyStore(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (Exception e) { + throw new RuntimeException("Failed setting up key store for " + pemKeyStore.keyPath() + ", " + pemKeyStore.certificatePath(), e); + } + } + + private static <T extends InnerNode> void setStringArrayParameter(SslContextFactory sslContextFactory, + List<T> configValues, + Function<T, String> nameProperty, + BiConsumer<SslContextFactory, String[]> setter) { + if (!configValues.isEmpty()) { + String[] nameArray = configValues.stream().map(nameProperty).toArray(String[]::new); + setter.accept(sslContextFactory, nameArray); + } + } + +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/package-info.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/package-info.java new file mode 100644 index 00000000000..f337e9d010b --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.jdisc.http.ssl.impl; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/package-info.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/package-info.java index 5f817d4cfc2..085e9dedf20 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/package-info.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/package-info.java @@ -2,7 +2,9 @@ /** * @author bjorncs */ +@PublicApi @ExportPackage package com.yahoo.jdisc.http.ssl; +import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def index 9ae4713c633..157ffabdd63 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def @@ -44,6 +44,23 @@ tcpNoDelay bool default=true # Whether to enable SSL for this connector. ssl.enabled bool default=false +# File with private key in PEM format +ssl.privateKeyFile string default="" + +# File with certificate in PEM format +ssl.certificateFile string default="" + +# with trusted CA certificates in PEM format. Used to verify clients +ssl.caCertificateFile string default="" + +# Client authentication mode. See SSLEngine.getNeedClientAuth()/getWantClientAuth() for details. +ssl.clientAuth enum { DISABLED, WANT_AUTH, NEED_AUTH } default=DISABLED + + +######################################################################################### +# Config below is deprecated. Do not use +######################################################################################### + # The name of the key to the password to the key store if in the secret store, if JKS is used. # Must be empty with PEM # By default this is also used to look up the password to the trust store. @@ -89,11 +106,9 @@ ssl.sslKeyManagerFactoryAlgorithm string default="SunX509" # The SSL protocol passed to SSLContext.getInstance() ssl.protocol string default="TLS" -# Client authentication mode. See SSLEngine.getNeedClientAuth()/getWantClientAuth() for details. -ssl.clientAuth enum { DISABLED, WANT_AUTH, NEED_AUTH } default=DISABLED - # The SecureRandom implementation passed to SSLEngine.init() # Java have a default pseudo-random number generator (PRNG) for crypto operations. This default may have performance # issues on some platform (e.g. NativePRNG in Linux utilizes a global lock). Changing the generator to SHA1PRNG may # improve performance. Set value to empty string to use the default generator. ssl.prng string default="" + diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/JksKeyStore.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/JksKeyStore.java deleted file mode 100644 index 1c7a917c688..00000000000 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/JksKeyStore.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http; - -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.KeyStore; - -/** - * @author Tony Vaagenes - * @author bjorncs - */ -public class JksKeyStore { - - private static final String KEY_STORE_TYPE = "JKS"; - - private final Path keyStoreFile; - private final String keyStorePassword; - - public JksKeyStore(Path keyStoreFile) { - this(keyStoreFile, null); - } - - public JksKeyStore(Path keyStoreFile, String keyStorePassword) { - this.keyStoreFile = keyStoreFile; - this.keyStorePassword = keyStorePassword; - } - - public String getKeyStorePassword() { - return keyStorePassword; - } - - public KeyStore loadJavaKeyStore() throws Exception { - try(InputStream stream = Files.newInputStream(keyStoreFile)) { - KeyStore keystore = KeyStore.getInstance(KEY_STORE_TYPE); - keystore.load(stream, keyStorePassword != null ? keyStorePassword.toCharArray() : null); - return keystore; - } - } - -} diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/SslContextFactory.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/SslContextFactory.java deleted file mode 100644 index d86516df453..00000000000 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/SslContextFactory.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http; - -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManagerFactory; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author Charles Kim - */ -public class SslContextFactory { - - private static final Logger log = Logger.getLogger(SslContextFactory.class.getName()); - private static final String DEFAULT_ALGORITHM = "SunX509"; - private static final String DEFAULT_PROTOCOL = "TLS"; - private final SSLContext sslContext; - - private SslContextFactory(SSLContext sslContext) { - this.sslContext = sslContext; - } - - public SSLContext getServerSSLContext() { - return this.sslContext; - } - - public static SslContextFactory newInstanceFromTrustStore(JksKeyStore trustStore) { - return newInstance(DEFAULT_ALGORITHM, DEFAULT_PROTOCOL, null, trustStore); - } - - public static SslContextFactory newInstance(JksKeyStore trustStore, JksKeyStore keyStore) { - return newInstance(DEFAULT_ALGORITHM, DEFAULT_PROTOCOL, keyStore, trustStore); - } - - public static SslContextFactory newInstance(String sslAlgorithm, String sslProtocol, - JksKeyStore keyStore, JksKeyStore trustStore) { - log.fine("Configuring SSLContext..."); - log.fine("Using " + sslAlgorithm + " algorithm."); - try { - SSLContext sslContext = SSLContext.getInstance(sslProtocol); - sslContext.init( - keyStore == null ? null : getKeyManagers(keyStore, sslAlgorithm), - trustStore == null ? null : getTrustManagers(trustStore, sslAlgorithm), - null); - return new SslContextFactory(sslContext); - } catch (Exception e) { - log.log(Level.SEVERE, "Got exception creating SSLContext.", e); - throw new RuntimeException(e); - } - } - - /** - * Used for the key store, which contains the SSL cert and private key. - */ - public static javax.net.ssl.KeyManager[] getKeyManagers(JksKeyStore keyStore, - String sslAlgorithm) throws Exception { - - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(sslAlgorithm); - String keyStorePassword = keyStore.getKeyStorePassword(); - keyManagerFactory.init( - keyStore.loadJavaKeyStore(), - keyStorePassword != null ? keyStorePassword.toCharArray() : null); - log.fine("KeyManagerFactory initialized with keystore"); - return keyManagerFactory.getKeyManagers(); - } - - /** - * Used for the trust store, which contains certificates from other parties that you expect to communicate with, - * or from Certificate Authorities that you trust to identify other parties. - */ - public static javax.net.ssl.TrustManager[] getTrustManagers(JksKeyStore trustStore, - String sslAlgorithm) - throws Exception { - - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(sslAlgorithm); - trustManagerFactory.init(trustStore.loadJavaKeyStore()); - log.fine("TrustManagerFactory initialized with truststore."); - return trustManagerFactory.getTrustManagers(); - } - -} diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/filter/SecurityRequestFilterChainTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/filter/SecurityRequestFilterChainTest.java new file mode 100644 index 00000000000..2e072f29039 --- /dev/null +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/filter/SecurityRequestFilterChainTest.java @@ -0,0 +1,145 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.filter; + +import com.yahoo.jdisc.AbstractResource; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ResponseDispatch; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.jdisc.test.TestDriver; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import static org.testng.Assert.assertEquals; + +/** + * @author bjorncs + */ +public class SecurityRequestFilterChainTest { + + + private static HttpRequest newRequest(URI uri, HttpRequest.Method method, HttpRequest.Version version) { + InetSocketAddress address = new InetSocketAddress("java.corp.yahoo.com", 69); + TestDriver driver = TestDriver.newSimpleApplicationInstanceWithoutOsgi(); + driver.activateContainer(driver.newContainerBuilder()); + HttpRequest request = HttpRequest.newServerRequest(driver, uri, method, version, address); + request.release(); + Assert.assertTrue(driver.close()); + return request; + } + + @Test + public void testFilterChainConstruction() { + SecurityRequestFilterChain chain = (SecurityRequestFilterChain)SecurityRequestFilterChain.newInstance(); + assertEquals(chain.getFilters().size(),0); + + List<SecurityRequestFilter> requestFilters = new ArrayList<SecurityRequestFilter>(); + chain = (SecurityRequestFilterChain)SecurityRequestFilterChain.newInstance(); + + chain = (SecurityRequestFilterChain)SecurityRequestFilterChain.newInstance(new RequestHeaderFilter("abc", "xyz"), + new RequestHeaderFilter("pqr", "def")); + + assertEquals(chain instanceof SecurityRequestFilterChain, true); + } + + + @Test + public void testFilterChainRun() { + RequestFilter chain = SecurityRequestFilterChain.newInstance(new RequestHeaderFilter("abc", "xyz"), + new RequestHeaderFilter("pqr", "def")); + + assertEquals(chain instanceof SecurityRequestFilterChain, true); + ResponseHandler handler = newResponseHandler(); + HttpRequest request = newRequest(URI.create("http://test/test"), HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); + chain.filter(request, handler); + Assert.assertTrue(request.headers().contains("abc", "xyz")); + Assert.assertTrue(request.headers().contains("pqr", "def")); + } + + @Test + public void testFilterChainResponds() { + RequestFilter chain = SecurityRequestFilterChain.newInstance( + new MyFilter(), + new RequestHeaderFilter("abc", "xyz"), + new RequestHeaderFilter("pqr", "def")); + + assertEquals(chain instanceof SecurityRequestFilterChain, true); + ResponseHandler handler = newResponseHandler(); + HttpRequest request = newRequest(URI.create("http://test/test"), HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); + chain.filter(request, handler); + Response response = getResponse(handler); + Assert.assertNotNull(response); + Assert.assertTrue(!request.headers().contains("abc", "xyz")); + Assert.assertTrue(!request.headers().contains("pqr", "def")); + } + + private class RequestHeaderFilter extends AbstractResource implements SecurityRequestFilter { + + private final String key; + private final String val; + + public RequestHeaderFilter(String key, String val) { + this.key = key; + this.val = val; + } + + @Override + public void filter(DiscFilterRequest request, ResponseHandler handler) { + request.setHeaders(key, val); + } + } + + private class MyFilter extends AbstractResource implements SecurityRequestFilter { + + @Override + public void filter(DiscFilterRequest request, ResponseHandler handler) { + ResponseDispatch.newInstance(Response.Status.FORBIDDEN).dispatch(handler); + } + } + + private static ResponseHandler newResponseHandler() { + return new NonWorkingResponseHandler(); + } + + private static Response getResponse(ResponseHandler handler) { + return ((NonWorkingResponseHandler) handler).getResponse(); + } + + private static class NonWorkingResponseHandler implements ResponseHandler { + + private Response response = null; + + @Override + public ContentChannel handleResponse(Response response) { + this.response = response; + return new NonWorkingContentChannel(); + } + + public Response getResponse() { + return response; + } + } + + private static class NonWorkingContentChannel implements ContentChannel { + + @Override + public void close(CompletionHandler handler) { + + } + + @Override + public void write(ByteBuffer buf, CompletionHandler handler) { + + } + + } + +} diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/filter/SecurityResponseFilterChainTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/filter/SecurityResponseFilterChainTest.java new file mode 100644 index 00000000000..b38ca240a78 --- /dev/null +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/filter/SecurityResponseFilterChainTest.java @@ -0,0 +1,75 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.filter; + +import com.yahoo.jdisc.AbstractResource; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.jdisc.http.HttpResponse; +import com.yahoo.jdisc.test.TestDriver; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.net.InetSocketAddress; +import java.net.URI; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +/** + * @author bjorncs + */ +public class SecurityResponseFilterChainTest { + private static HttpRequest newRequest(URI uri, HttpRequest.Method method, HttpRequest.Version version) { + InetSocketAddress address = new InetSocketAddress("java.corp.yahoo.com", 69); + TestDriver driver = TestDriver.newSimpleApplicationInstanceWithoutOsgi(); + driver.activateContainer(driver.newContainerBuilder()); + HttpRequest request = HttpRequest.newServerRequest(driver, uri, method, version, address); + request.release(); + Assert.assertTrue(driver.close()); + return request; + } + + @Test + public void testFilterChainConstruction() { + SecurityResponseFilterChain chain = (SecurityResponseFilterChain)SecurityResponseFilterChain.newInstance(); + assertEquals(chain.getFilters().size(),0); + + chain = (SecurityResponseFilterChain)SecurityResponseFilterChain.newInstance(new ResponseHeaderFilter("abc", "xyz"), + new ResponseHeaderFilter("pqr", "def")); + + assertEquals(chain instanceof SecurityResponseFilterChain, true); + } + + @Test + public void testFilterChainRun() { + URI uri = URI.create("http://localhost:8080/echo"); + HttpRequest request = newRequest(uri, HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); + Response response = HttpResponse.newInstance(Response.Status.OK); + + ResponseFilter chain = SecurityResponseFilterChain.newInstance(new ResponseHeaderFilter("abc", "xyz"), + new ResponseHeaderFilter("pqr", "def")); + chain.filter(response, null); + assertTrue(response.headers().contains("abc", "xyz")); + assertTrue(response.headers().contains("pqr", "def")); + } + + private class ResponseHeaderFilter extends AbstractResource implements SecurityResponseFilter { + + private final String key; + private final String val; + + public ResponseHeaderFilter(String key, String val) { + this.key = key; + this.val = val; + } + + @Override + public void filter(DiscFilterResponse response, RequestView request) { + response.setHeaders(key, val); + } + + } + + + +} diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java index d1a78f33e8f..a4baccb86c9 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java @@ -10,9 +10,7 @@ import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ConnectorConfig.Builder; import com.yahoo.jdisc.http.server.jetty.ConnectorFactory; -import com.yahoo.jdisc.http.server.jetty.TestDrivers; -import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator; -import com.yahoo.jdisc.http.ssl.DefaultSslTrustStoreConfigurator; +import com.yahoo.jdisc.http.ssl.impl.DefaultSslContextFactoryProvider; /** * Guice module for test ConnectorFactories @@ -48,19 +46,7 @@ public class ConnectorFactoryRegistryModule implements Module { private static class StaticKeyDbConnectorFactory extends ConnectorFactory { public StaticKeyDbConnectorFactory(ConnectorConfig connectorConfig) { - super(connectorConfig, - new DefaultSslKeyStoreConfigurator(connectorConfig, new MockSecretStore()), - new DefaultSslTrustStoreConfigurator(connectorConfig, new MockSecretStore())); - } - - } - - @SuppressWarnings("deprecation") - private static final class MockSecretStore implements com.yahoo.jdisc.http.SecretStore { - - @Override - public String getSecret(String key) { - return TestDrivers.KEY_STORE_PASSWORD; + super(connectorConfig, new DefaultSslContextFactoryProvider(connectorConfig)); } } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java index 083be36043e..eb18a3ee341 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java @@ -3,8 +3,7 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator; -import com.yahoo.jdisc.http.ssl.DefaultSslTrustStoreConfigurator; +import com.yahoo.jdisc.http.ssl.impl.DefaultSslContextFactoryProvider; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.AbstractHandler; @@ -106,10 +105,7 @@ public class ConnectorFactoryTest { } private static ConnectorFactory createConnectorFactory(ConnectorConfig config) { - ThrowingSecretStore secretStore = new ThrowingSecretStore(); - return new ConnectorFactory(config, - new DefaultSslKeyStoreConfigurator(config, secretStore), - new DefaultSslTrustStoreConfigurator(config, secretStore)); + return new ConnectorFactory(config, new DefaultSslContextFactoryProvider(config)); } private static class HelloWorldHandler extends AbstractHandler { @@ -138,14 +134,4 @@ public class ConnectorFactoryTest { private static class DummyContext implements Metric.Context { } - @SuppressWarnings("deprecation") - private static final class ThrowingSecretStore implements com.yahoo.jdisc.http.SecretStore { - - @Override - public String getSecret(String key) { - throw new UnsupportedOperationException("A secret store is not available"); - } - - } - } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java index 39b68fcf1f6..227b0b20f10 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java @@ -1,20 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; -import com.google.inject.Key; import com.google.inject.Module; import com.yahoo.jdisc.application.ContainerBuilder; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.SslContextFactory; -import com.yahoo.jdisc.http.JksKeyStore; +import com.yahoo.security.SslContextBuilder; import javax.net.ssl.SSLContext; import java.io.IOException; import java.nio.file.Paths; -import static com.google.inject.name.Names.named; - /** * This class is based on the class by the same name in the jdisc_http_service module. * It provides functionality for setting up a jdisc container with an HTTP server and handlers. @@ -61,9 +57,7 @@ public class TestDriver { public SimpleHttpClient client() { return client; } - public SimpleHttpClient newClient() throws IOException { return newClient(false); } - - public SimpleHttpClient newClient(final boolean useCompression) throws IOException { + public SimpleHttpClient newClient(final boolean useCompression) { return new SimpleHttpClient(newSslContext(), server.getListenPort(), useCompression); } @@ -75,10 +69,10 @@ public class TestDriver { ConnectorConfig.Ssl sslConfig = builder.getInstance(ConnectorConfig.class).ssl(); if (!sslConfig.enabled()) return null; - JksKeyStore keyStore = new JksKeyStore( - Paths.get(sslConfig.keyStorePath()), - builder.getInstance(Key.get(String.class, named("keyStorePassword")))); - return SslContextFactory.newInstanceFromTrustStore(keyStore).getServerSSLContext(); + return new SslContextBuilder() + .withKeyStore(Paths.get(sslConfig.privateKeyFile()), Paths.get(sslConfig.certificateFile())) + .withTrustStore(Paths.get(sslConfig.caCertificateFile())) + .build(); } } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java index f4344545637..b7805328124 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java @@ -17,15 +17,13 @@ import com.yahoo.jdisc.http.server.FilterBindings; import java.io.IOException; -import static com.google.inject.name.Names.named; - /** * @author Simon Thoresen Hult */ public class TestDrivers { - private static final String KEY_STORE = "src/test/resources/ssl_keystore_test.jks"; - public static final String KEY_STORE_PASSWORD = "secret"; + private static final String PRIVATE_KEY_FILE = "src/test/resources/pem/test.key"; + private static final String CERTIFICATE_FILE = "src/test/resources/pem/test.crt"; public static TestDriver newConfiguredInstance(final RequestHandler requestHandler, final ServerConfig.Builder serverConfig, @@ -59,18 +57,10 @@ public class TestDrivers { new ConnectorConfig.Builder() .ssl(new ConnectorConfig.Ssl.Builder() .enabled(true) - .keyDbKey("dummy-key-for-StaticKeyDbConnectorFactory.getPasswordFromKeydb") - .keyStorePath(KEY_STORE) - .trustStorePath(KEY_STORE)), - Modules.combine(new AbstractModule() { - - @Override - protected void configure() { - bind(String.class).annotatedWith(named("keyStorePassword")) - .toInstance(KEY_STORE_PASSWORD); - } - }, Modules.combine(guiceModules)) - )); + .privateKeyFile(PRIVATE_KEY_FILE) + .certificateFile(CERTIFICATE_FILE) + .caCertificateFile(CERTIFICATE_FILE)), + Modules.combine(guiceModules))); } private static Module newConfigModule( diff --git a/jdisc_http_service/src/test/resources/ssl_keystore_test.jks b/jdisc_http_service/src/test/resources/ssl_keystore_test.jks Binary files differdeleted file mode 100644 index 6dbb19b9692..00000000000 --- a/jdisc_http_service/src/test/resources/ssl_keystore_test.jks +++ /dev/null diff --git a/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java b/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java index 8794bd507a2..e54279e0541 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java @@ -50,11 +50,6 @@ public final class ErrorCode { /** No services found for the message route. */ public static final int NO_SERVICES_FOR_ROUTE = FATAL_ERROR + 3; - /** The selected service was out of service. - */ - @Deprecated // Unused and will be removed - public static final int SERVICE_OOS = FATAL_ERROR + 4; - /** An error occured while encoding the message. */ public static final int ENCODE_ERROR = FATAL_ERROR + 5; @@ -118,7 +113,6 @@ public final class ErrorCode { case SEND_QUEUE_CLOSED : return "SEND_QUEUE_CLOSED"; case SEND_QUEUE_FULL : return "SEND_QUEUE_FULL"; case SEQUENCE_ERROR : return "SEQUENCE_ERROR"; - case SERVICE_OOS : return "SERVICE_OOS"; case SESSION_BUSY : return "SESSION_BUSY"; case TIMEOUT : return "TIMEOUT"; case TRANSIENT_ERROR : return "TRANSIENT_ERROR"; diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index d2fca309a19..7bea2d0825a 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -81,11 +81,11 @@ public class RankProfilesConfigImporter { referencedFunctions.put(reference.get(), new ExpressionFunction(reference.get().serialForm(), arguments, expression)); } - else if (property.name().equals("vespa.rank.firstphase")) { // Include in addition to macros + else if (property.name().equals("vespa.rank.firstphase")) { // Include in addition to functions firstPhase = new ExpressionFunction("firstphase", new ArrayList<>(), new RankingExpression("first-phase", property.value())); } - else if (property.name().equals("vespa.rank.secondphase")) { // Include in addition to macros + else if (property.name().equals("vespa.rank.secondphase")) { // Include in addition to functions secondPhase = new ExpressionFunction("secondphase", new ArrayList<>(), new RankingExpression("second-phase", property.value())); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index 1febe070072..9259b522d17 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -52,6 +52,8 @@ public class AclMaintainer implements Runnable { private void applyRedirect(Container container, InetAddress address) { IPVersion ipVersion = IPVersion.get(address); + // Necessary to avoid the routing packets destined for the node's own public IP address + // via the bridge, which is illegal. String redirectRule = "-A OUTPUT -d " + InetAddresses.toAddrString(address) + ipVersion.singleHostCidr() + " -j REDIRECT"; IPTablesEditor.editLogOnError(dockerOperations, container.name, ipVersion, "nat", NatTableLineEditor.from(redirectRule)); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java index ca79e8bb113..43c9f7729ef 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java @@ -13,23 +13,23 @@ import java.util.Optional; class FileContentCache { private final UnixPath path; - private Optional<String> value = Optional.empty(); + private Optional<byte[]> value = Optional.empty(); private Optional<Instant> modifiedTime = Optional.empty(); FileContentCache(UnixPath path) { this.path = path; } - String get(Instant lastModifiedTime) { + byte[] get(Instant lastModifiedTime) { if (!value.isPresent() || lastModifiedTime.compareTo(modifiedTime.get()) > 0) { - value = Optional.of(path.readUtf8File()); + value = Optional.of(path.readBytes()); modifiedTime = Optional.of(lastModifiedTime); } return value.get(); } - void updateWith(String content, Instant modifiedTime) { + void updateWith(byte[] content, Instant modifiedTime) { this.value = Optional.of(content); this.modifiedTime = Optional.of(modifiedTime); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java index 7f11707c1cc..78f720074dc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java @@ -5,7 +5,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import java.nio.file.Path; -import java.util.Objects; +import java.util.Arrays; import java.util.Optional; import java.util.logging.Logger; @@ -46,7 +46,7 @@ public class FileSync { } private boolean maybeUpdateContent(TaskContext taskContext, - Optional<String> content, + Optional<byte[]> content, FileAttributesCache currentAttributes) { if (!content.isPresent()) { return false; @@ -55,16 +55,16 @@ public class FileSync { if (!currentAttributes.exists()) { taskContext.recordSystemModification(logger, "Creating file " + path); path.createParents(); - path.writeUtf8File(content.get()); + path.writeBytes(content.get()); contentCache.updateWith(content.get(), currentAttributes.forceGet().lastModifiedTime()); return true; } - if (Objects.equals(content.get(), contentCache.get(currentAttributes.get().lastModifiedTime()))) { + if (Arrays.equals(content.get(), contentCache.get(currentAttributes.get().lastModifiedTime()))) { return false; } else { taskContext.recordSystemModification(logger, "Patching file " + path); - path.writeUtf8File(content.get()); + path.writeBytes(content.get()); contentCache.updateWith(content.get(), currentAttributes.forceGet().lastModifiedTime()); return true; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java index c41fd71c62c..f7eba68e455 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.function.Supplier; @@ -16,11 +17,15 @@ public class FileWriter { private final Path path; private final FileSync fileSync; private final PartialFileData.Builder fileDataBuilder = PartialFileData.builder(); - private final Supplier<String> contentProducer; + private final Supplier<byte[]> contentProducer; private boolean overwriteExistingFile = true; public FileWriter(Path path, Supplier<String> contentProducer) { + this(path, () -> contentProducer.get().getBytes(StandardCharsets.UTF_8)); + } + + public FileWriter(Path path, ByteArraySupplier contentProducer) { this.path = path; this.fileSync = new FileSync(path); this.contentProducer = contentProducer; @@ -55,4 +60,7 @@ public class FileWriter { PartialFileData fileData = fileDataBuilder.create(); return fileSync.convergeTo(context, fileData); } + + @FunctionalInterface + public interface ByteArraySupplier extends Supplier<byte[]> { } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java index b931a374230..0fae9b17eba 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Optional; /** @@ -11,7 +13,7 @@ import java.util.Optional; */ // @Immutable public class PartialFileData { - private final Optional<String> content; + private final Optional<byte[]> content; private final Optional<String> owner; private final Optional<String> group; private final Optional<String> permissions; @@ -20,17 +22,17 @@ public class PartialFileData { return new Builder(); } - public PartialFileData(Optional<String> content, - Optional<String> owner, - Optional<String> group, - Optional<String> permissions) { + private PartialFileData(Optional<byte[]> content, + Optional<String> owner, + Optional<String> group, + Optional<String> permissions) { this.content = content; this.owner = owner; this.group = group; this.permissions = permissions; } - public Optional<String> getContent() { + public Optional<byte[]> getContent() { return content; } @@ -47,12 +49,14 @@ public class PartialFileData { } public static class Builder { - private Optional<String> content = Optional.empty(); + private Optional<byte[]> content = Optional.empty(); private Optional<String> owner = Optional.empty(); private Optional<String> group = Optional.empty(); private Optional<String> permissions = Optional.empty(); - public Builder withContent(String content) { this.content = Optional.of(content); return this; } + public Builder withContent(byte[] content) { this.content = Optional.of(content); return this; } + public Builder withContent(String content, Charset charset) { return withContent(content.getBytes(charset)); } + public Builder withContent(String content) { return withContent(content, StandardCharsets.UTF_8); } public Builder withOwner(String owner) { this.owner = Optional.of(owner); return this; } public Builder withGroup(String group) { this.group = Optional.of(group); return this; } public Builder withPermissions(String permissions) { this.permissions = Optional.of(permissions); return this; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 90404bb596e..4baba9acb4e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -52,13 +52,19 @@ public class UnixPath { } public String readUtf8File() { - byte[] byteContent = uncheck(() -> Files.readAllBytes(path)); - return new String(byteContent, StandardCharsets.UTF_8); + return new String(readBytes(), StandardCharsets.UTF_8); + } + + public byte[] readBytes() { + return uncheck(() -> Files.readAllBytes(path)); } public void writeUtf8File(String content, OpenOption... options) { - byte[] contentInUtf8 = content.getBytes(StandardCharsets.UTF_8); - uncheck(() -> Files.write(path, contentInUtf8, options)); + writeBytes(content.getBytes(StandardCharsets.UTF_8), options); + } + + public void writeBytes(byte[] content, OpenOption... options) { + uncheck(() -> Files.write(path, content, options)); } public String getPermissions() { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java index 351856c4852..b61ebb610af 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java @@ -41,6 +41,12 @@ public class SystemCtl { this.terminal = terminal; } + public void daemonReload(TaskContext taskContext) { + terminal.newCommandLine(taskContext) + .add("systemctl", "daemon-reload") + .execute(); + } + public SystemCtlEnable enable(String unit) { return new SystemCtlEnable(unit); } public SystemCtlDisable disable(String unit) { return new SystemCtlDisable(unit); } public SystemCtlStart start(String unit) { return new SystemCtlStart(unit); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java index 83ab5462fb6..f499d8b46ad 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java @@ -4,9 +4,10 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.time.Instant; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertArrayEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -17,41 +18,44 @@ public class FileContentCacheTest { private final UnixPath unixPath = mock(UnixPath.class); private final FileContentCache cache = new FileContentCache(unixPath); + private final byte[] content = "content".getBytes(StandardCharsets.UTF_8); + private final byte[] newContent = "new-content".getBytes(StandardCharsets.UTF_8); + @Test public void get() { - when(unixPath.readUtf8File()).thenReturn("content"); - assertEquals("content", cache.get(Instant.ofEpochMilli(0))); - verify(unixPath, times(1)).readUtf8File(); + when(unixPath.readBytes()).thenReturn(content); + assertArrayEquals(content, cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1)).readBytes(); verifyNoMoreInteractions(unixPath); // cache hit - assertEquals("content", cache.get(Instant.ofEpochMilli(0))); - verify(unixPath, times(1)).readUtf8File(); + assertArrayEquals(content, cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1)).readBytes(); verifyNoMoreInteractions(unixPath); // cache miss - when(unixPath.readUtf8File()).thenReturn("new-content"); - assertEquals("new-content", cache.get(Instant.ofEpochMilli(1))); - verify(unixPath, times(1 + 1)).readUtf8File(); + when(unixPath.readBytes()).thenReturn(newContent); + assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(1))); + verify(unixPath, times(1 + 1)).readBytes(); verifyNoMoreInteractions(unixPath); // cache hit both at times 0 and 1 - assertEquals("new-content", cache.get(Instant.ofEpochMilli(0))); - verify(unixPath, times(1 + 1)).readUtf8File(); + assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1 + 1)).readBytes(); verifyNoMoreInteractions(unixPath); - assertEquals("new-content", cache.get(Instant.ofEpochMilli(1))); - verify(unixPath, times(1 + 1)).readUtf8File(); + assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(1))); + verify(unixPath, times(1 + 1)).readBytes(); verifyNoMoreInteractions(unixPath); } @Test public void updateWith() { - cache.updateWith("content", Instant.ofEpochMilli(2)); - assertEquals("content", cache.get(Instant.ofEpochMilli(2))); + cache.updateWith(content, Instant.ofEpochMilli(2)); + assertArrayEquals(content, cache.get(Instant.ofEpochMilli(2))); verifyNoMoreInteractions(unixPath); - cache.updateWith("new-content", Instant.ofEpochMilli(4)); - assertEquals("new-content", cache.get(Instant.ofEpochMilli(4))); + cache.updateWith(newContent, Instant.ofEpochMilli(4)); + assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(4))); verifyNoMoreInteractions(unixPath); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java index dbc2cc9a5d5..a141faf290b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java @@ -14,6 +14,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -64,7 +65,7 @@ public class FileSyncTest { assertTrue(fileSync.convergeTo(taskContext, fileData)); assertTrue(Files.isRegularFile(path)); - fileData.getContent().ifPresent(content -> assertEquals(content, unixPath.readUtf8File())); + fileData.getContent().ifPresent(content -> assertArrayEquals(content, unixPath.readBytes())); fileData.getOwner().ifPresent(owner -> assertEquals(owner, unixPath.getOwner())); fileData.getGroup().ifPresent(group -> assertEquals(group, unixPath.getGroup())); fileData.getPermissions().ifPresent(permissions -> assertEquals(permissions, unixPath.getPermissions())); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index e387fb2d0ed..39a2787ca9b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -234,7 +234,6 @@ public class MetricsReporter extends Maintainer { for (Flavor flavor : dockerFlavors) { Metric.Context context = getContextAt("flavor", flavor.name()); metric.set("hostedVespa.docker.freeCapacityFlavor", capacity.freeCapacityInFlavorEquivalence(flavor), context); - metric.set("hostedVespa.docker.idealHeadroomFlavor", flavor.getIdealHeadroom(), context); metric.set("hostedVespa.docker.hostsAvailableFlavor", capacity.getNofHostsAvailableFor(flavor), context); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index efa1cd3745d..6168d6fcf78 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -48,6 +48,8 @@ public class CapacityPolicies { return flavors.getFlavorOrThrow(requestedFlavor.get()); String defaultFlavorName = zone.defaultFlavor(cluster.type()); + if (zone.system() == SystemName.cd) + return flavors.getFlavorOrThrow(requestedFlavor.orElse(defaultFlavorName)); switch(zone.environment()) { case dev : case test : case staging : return flavors.getFlavorOrThrow(defaultFlavorName); default : return flavors.getFlavorOrThrow(requestedFlavor.orElse(defaultFlavorName)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java index b52506c268c..cff62508ec6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java @@ -19,7 +19,7 @@ public class FlavorConfigBuilder { return new FlavorsConfig(builder); } - public FlavorsConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type, int headRoom) { + public FlavorsConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); flavor.name(flavorName); flavor.description("Flavor-name-is-" + flavorName); @@ -27,15 +27,10 @@ public class FlavorConfigBuilder { flavor.minCpuCores(cpu); flavor.minMainMemoryAvailableGb(mem); flavor.environment(type.name()); - flavor.idealHeadroom(headRoom); builder.flavor(flavor); return flavor; } - public FlavorsConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { - return addFlavor(flavorName, cpu, mem, disk, type, 0); - } - public FlavorsConfig.Flavor.Builder addNonStockFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); flavor.name(flavorName); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index dff6378a19a..eb3be9e6d80 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -52,7 +52,6 @@ public class GroupPreparer { application, cluster, requestedNodes, - nodeRepository.getAvailableFlavors(), spareCount, nodeRepository.nameResolver()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index b48d69447a8..a74f3b2d116 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.provision.Node; @@ -15,7 +14,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -44,17 +42,15 @@ public class NodePrioritizer { private final boolean isDocker; private final boolean isAllocatingForReplacement; private final Set<Node> spareHosts; - private final Map<Node, ResourceCapacity> headroomHosts; NodePrioritizer(List<Node> allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, - NodeFlavors nodeFlavors, int spares, NameResolver nameResolver) { + int spares, NameResolver nameResolver) { this.allNodes = Collections.unmodifiableList(allNodes); this.requestedNodes = nodeSpec; this.clusterSpec = clusterSpec; this.appId = appId; this.nameResolver = nameResolver; this.spareHosts = findSpareHosts(allNodes, spares); - this.headroomHosts = findHeadroomHosts(allNodes, spareHosts, nodeFlavors); this.capacity = new DockerHostCapacity(allNodes); @@ -93,62 +89,6 @@ public class NodePrioritizer { } /** - * Headroom hosts are the host with the least but sufficient capacity for the requested headroom. - * - * If not enough headroom - the headroom violating hosts are the once that are closest to fulfill - * a headroom request. - */ - private static Map<Node, ResourceCapacity> findHeadroomHosts(List<Node> nodes, Set<Node> spareNodes, NodeFlavors flavors) { - DockerHostCapacity capacity = new DockerHostCapacity(nodes); - Map<Node, ResourceCapacity> headroomHosts = new HashMap<>(); - - List<Node> hostsSortedOnLeastCapacity = nodes.stream() - .filter(n -> !spareNodes.contains(n)) - .filter(node -> node.type().equals(NodeType.host)) - .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) - .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0) - .sorted((a, b) -> capacity.compareWithoutInactive(b, a)) - .collect(Collectors.toList()); - - // For all flavors with ideal headroom - find which hosts this headroom should be allocated to - for (Flavor flavor : flavors.getFlavors().stream().filter(f -> f.getIdealHeadroom() > 0).collect(Collectors.toList())) { - Set<Node> tempHeadroom = new HashSet<>(); - Set<Node> notEnoughCapacity = new HashSet<>(); - - ResourceCapacity headroomCapacity = ResourceCapacity.of(flavor); - - // Select hosts that has available capacity for both headroom and for new allocations - for (Node host : hostsSortedOnLeastCapacity) { - if (headroomHosts.containsKey(host)) continue; - if (capacity.hasCapacityWhenRetiredAndInactiveNodesAreGone(host, headroomCapacity)) { - headroomHosts.put(host, headroomCapacity); - tempHeadroom.add(host); - } else { - notEnoughCapacity.add(host); - } - - if (tempHeadroom.size() == flavor.getIdealHeadroom()) { - break; - } - } - - // Now check if we have enough headroom - if not choose the nodes that almost has it - if (tempHeadroom.size() < flavor.getIdealHeadroom()) { - List<Node> violations = notEnoughCapacity.stream() - .sorted((a, b) -> capacity.compare(b, a)) - .limit(flavor.getIdealHeadroom() - tempHeadroom.size()) - .collect(Collectors.toList()); - - for (Node hostViolatingHeadrom : violations) { - headroomHosts.put(hostViolatingHeadrom, headroomCapacity); - } - } - } - - return headroomHosts; - } - - /** * @return The list of nodes sorted by PrioritizableNode::compare */ List<PrioritizableNode> prioritize() { @@ -257,16 +197,6 @@ public class NodePrioritizer { if (spareHosts.contains(parent)) { pri.violatesSpares = true; } - - if (headroomHosts.containsKey(parent) && isPreferredNodeToBeReloacted(allNodes, node, parent)) { - ResourceCapacity neededCapacity = headroomHosts.get(parent); - - // If the node is new then we need to check the headroom requirement after it has been added - if (isNewNode) { - neededCapacity = ResourceCapacity.composite(neededCapacity, new ResourceCapacity(node)); - } - pri.violatesHeadroom = !capacity.hasCapacity(parent, neededCapacity); - } } return pri; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java index 40f76125064..d05f4e957c3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java @@ -14,7 +14,7 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { Node node; - /** The free capacity excluding headroom, including retired allocations */ + /** The free capacity, including retired allocations */ ResourceCapacity freeParentCapacity = new ResourceCapacity(); /** The parent host (docker or hypervisor) */ @@ -23,9 +23,6 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { /** True if the node is allocated to a host that should be dedicated as a spare */ boolean violatesSpares; - /** True if the node is (or would be) allocated on slots that should be dedicated to headroom */ - boolean violatesHeadroom; - /** True if this is a node that has been retired earlier in the allocation process */ boolean isSurplusNode; @@ -45,8 +42,6 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { // First always pick nodes without violation above nodes with violations if (!this.violatesSpares && other.violatesSpares) return -1; if (!other.violatesSpares && this.violatesSpares) return 1; - if (!this.violatesHeadroom && other.violatesHeadroom) return -1; - if (!other.violatesHeadroom && this.violatesHeadroom) return 1; // Choose active nodes if (this.node.state().equals(Node.State.active) && !other.node.state().equals(Node.State.active)) return -1; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java index 7157f893970..248299e7991 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java @@ -170,7 +170,6 @@ public class MetricsReporterTest { assertEquals(4.0, metric.values.get("hostedVespa.docker.freeCapacityCpu")); assertContext(metric, "hostedVespa.docker.freeCapacityFlavor", 1, 0); - assertContext(metric, "hostedVespa.docker.idealHeadroomFlavor", 0, 0); assertContext(metric, "hostedVespa.docker.hostsAvailableFlavor", 1l, 0l); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index 62212447c2e..bf93e114c8d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -44,8 +44,8 @@ public class AllocationSimulator { // Setup flavors // FlavorConfigBuilder b = new FlavorConfigBuilder(); - b.addFlavor("host-large", 8., 8., 8, Flavor.Type.BARE_METAL).idealHeadroom(1); - b.addFlavor("host-small", 5., 5., 5, Flavor.Type.BARE_METAL).idealHeadroom(2); + b.addFlavor("host-large", 8., 8., 8, Flavor.Type.BARE_METAL); + b.addFlavor("host-small", 5., 5., 5, Flavor.Type.BARE_METAL); b.addFlavor("d-1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER); b.addFlavor("d-2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER); b.addFlavor("d-3", 3, 3., 3, Flavor.Type.DOCKER_CONTAINER); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java index 3be56131a05..223a8bc83b0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java @@ -45,57 +45,10 @@ import static org.junit.Assert.fail; public class DynamicDockerProvisioningTest { /** - * Test relocation of nodes that violate headroom. - * <p> - * Setup 4 docker hosts and allocate one container on each (from two different applications) - * No spares - only headroom (4xd-2) - * <p> - * One application is now violating headroom and need relocation - * <p> - * Initial allocation of app 1 and 2 --> final allocation (headroom marked as H): - * <p> - * | H | H | H | H | | | | | | - * | H | H | H1a | H1b | --> | | | | | - * | | | 2a | 2b | | 1a | 1b | 2a | 2b | - */ - @Test - public void relocate_nodes_from_headroom_hosts() { - ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); - tester.makeReadyNodes(4, "host-small", NodeType.host, 32); - deployZoneApp(tester); - List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); - - // Application 1 - ApplicationId application1 = makeApplicationId("t1", "a1"); - ClusterSpec clusterSpec1 = clusterSpec("myContent.t1.a1"); - addAndAssignNode(application1, "1a", dockerHosts.get(2).hostname(), clusterSpec1, flavor, 0, tester); - addAndAssignNode(application1, "1b", dockerHosts.get(3).hostname(), clusterSpec1, flavor, 1, tester); - - // Application 2 - ApplicationId application2 = makeApplicationId("t2", "a2"); - ClusterSpec clusterSpec2 = clusterSpec("myContent.t2.a2"); - addAndAssignNode(application2, "2a", dockerHosts.get(2).hostname(), clusterSpec2, flavor, 0, tester); - addAndAssignNode(application2, "2b", dockerHosts.get(3).hostname(), clusterSpec2, flavor, 1, tester); - - // Redeploy one of the applications - deployapp(application1, clusterSpec1, flavor, tester, 2); - - // Assert that the nodes are spread across all hosts (to allow headroom) - Set<String> hostsWithChildren = new HashSet<>(); - for (Node node : tester.nodeRepository().getNodes(NodeType.tenant, Node.State.active)) { - if (!isInactiveOrRetired(node)) { - hostsWithChildren.add(node.parentHostname().get()); - } - } - Assert.assertEquals(4, hostsWithChildren.size()); - } - - /** * Test relocation of nodes from spare hosts. * <p> * Setup 4 docker hosts and allocate one container on each (from two different applications) - * No headroom defined - only getSpareCapacityProd() spares. + * getSpareCapacityProd() spares. * <p> * Check that it relocates containers away from the getSpareCapacityProd() spares * <p> @@ -141,132 +94,6 @@ public class DynamicDockerProvisioningTest { } /** - * Test that new docker nodes that will result in headroom violations are - * correctly marked as this. - * <p> - * When redeploying app1 - should not do anything (as moving app1 to host 0 and 1 would violate headroom). - * Then redeploy app 2 - should cause a relocation. - * <p> - * | H | H | H2a | H2b | | H | H | H | H | - * | H | H | H1a | H1b | --> | H | H | H1a | H1b | - * | | | 1a | 1b | | 2a | 2b | 1a | 1b | - */ - @Test - public void new_docker_nodes_are_marked_as_headroom_violations() { - ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); - tester.makeReadyNodes(4, "host-small", NodeType.host, 32); - deployZoneApp(tester); - List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - Flavor flavorD2 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-2"); - Flavor flavorD1 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); - - // Application 1 - ApplicationId application1 = makeApplicationId("t1", "1"); - ClusterSpec clusterSpec1 = clusterSpec("myContent.t1.a1"); - String hostParent2 = dockerHosts.get(2).hostname(); - String hostParent3 = dockerHosts.get(3).hostname(); - addAndAssignNode(application1, "1a", hostParent2, clusterSpec1, flavorD2, 0, tester); - addAndAssignNode(application1, "1b", hostParent3, clusterSpec1, flavorD2, 1, tester); - - // Application 2 - ApplicationId application2 = makeApplicationId("t2", "2"); - ClusterSpec clusterSpec2 = clusterSpec("myContent.t2.a2"); - addAndAssignNode(application2, "2a", hostParent2, clusterSpec2, flavorD1, 0, tester); - addAndAssignNode(application2, "2b", hostParent3, clusterSpec2, flavorD1, 1, tester); - - // Assert allocation placement - prior to re-deployment - assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); - assertApplicationHosts(tester.nodeRepository().getNodes(application2), hostParent2, hostParent3); - - // Redeploy application 1 - deployapp(application1, clusterSpec1, flavorD2, tester, 2); - - // Re-assert allocation placement - assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); - assertApplicationHosts(tester.nodeRepository().getNodes(application2), hostParent2, hostParent3); - - // Redeploy application 2 - deployapp(application2, clusterSpec2, flavorD1, tester, 2); - - // Now app2 should have re-located - assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); - assertApplicationHosts(tester.nodeRepository().getNodes(application2), dockerHosts.get(0).hostname(), dockerHosts.get(1).hostname()); - } - - /** - * Test that we only relocate the smallest nodes from a host to free up headroom. - * <p> - * The reason we want to do this is that it is an cheap approximation for the optimal solution as we - * pick headroom to be on the hosts were we are closest to fulfill the headroom requirement. - * - * Both applications could be moved here to free up headroom - but we want app2 (which is smallest) to be moved. - * <p> - * | H | H | H2a | H2b | | H | H | H | H | - * | H | H | H1a | H1b | --> | H | H | H | H | - * | | | 1a | 1b | | 2a | 2b | 1a | 1b | - * | | | | | | | | 1a | 1b | - */ - @Test - public void only_preferred_container_is_moved_from_hosts_with_headroom_violations() { - ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); - tester.makeReadyNodes(4, "host-medium", NodeType.host, 32); - deployZoneApp(tester); - List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - Flavor flavorD2 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-2"); - Flavor flavorD1 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); - - // Application 1 - ApplicationId application1 = makeApplicationId("t1", "1"); - ClusterSpec clusterSpec1 = clusterSpec("myContent.t1.a1"); - String hostParent2 = dockerHosts.get(2).hostname(); - String hostParent3 = dockerHosts.get(3).hostname(); - addAndAssignNode(application1, "1a", hostParent2, clusterSpec1, flavorD2, 0, tester); - addAndAssignNode(application1, "1b", hostParent3, clusterSpec1, flavorD2, 1, tester); - - // Application 2 - ApplicationId application2 = makeApplicationId("t2", "2"); - ClusterSpec clusterSpec2 = clusterSpec("myContent.t2.a2"); - addAndAssignNode(application2, "2a", hostParent2, clusterSpec2, flavorD1, 0, tester); - addAndAssignNode(application2, "2b", hostParent3, clusterSpec2, flavorD1, 1, tester); - - // Assert allocation placement - prior to re-deployment - assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); - assertApplicationHosts(tester.nodeRepository().getNodes(application2), hostParent2, hostParent3); - - // Redeploy application 1 - deployapp(application1, clusterSpec1, flavorD2, tester, 2); - - // Re-assert allocation placement - assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); - assertApplicationHosts(tester.nodeRepository().getNodes(application2), hostParent2, hostParent3); - - // Redeploy application 2 - deployapp(application2, clusterSpec2, flavorD1, tester, 2); - - // Now app2 should have re-located - assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); - assertApplicationHosts(tester.nodeRepository().getNodes(application2), dockerHosts.get(0).hostname(), dockerHosts.get(1).hostname()); - } - - private void assertApplicationHosts(List<Node> nodes, String... parents) { - for (Node node : nodes) { - // Ignore retired and non-active nodes - if (!node.state().equals(Node.State.active) || - node.allocation().get().membership().retired()) { - continue; - } - boolean found = false; - for (String parent : parents) { - if (node.parentHostname().get().equals(parent)) { - found = true; - break; - } - } - Assert.assertTrue(found); - } - } - - /** * Test an allocation workflow: * <p> * 5 Hosts of capacity 3 (2 spares) @@ -323,8 +150,7 @@ public class DynamicDockerProvisioningTest { /** * Test redeployment of nodes that violates spare headroom - but without alternatives * <p> - * Setup 2 docker hosts and allocate one app with a container on each - * No headroom defined - only 2 spares. + * Setup 2 docker hosts and allocate one app with a container on each. 2 spares * <p> * Initial allocation of app 1 --> final allocation: * <p> @@ -333,7 +159,7 @@ public class DynamicDockerProvisioningTest { * | 1a | 1b | | 1a | 1b | */ @Test - public void do_not_relocate_nodes_from_spare_if_no_where_to_reloacte_them() { + public void do_not_relocate_nodes_from_spare_if_no_where_to_relocate_them() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); tester.makeReadyNodes(2, "host-small", NodeType.host, 32); deployZoneApp(tester); @@ -475,16 +301,13 @@ public class DynamicDockerProvisioningTest { .collect(Collectors.toList()); } - private FlavorsConfig flavorsConfig(boolean includeHeadroom) { + private FlavorsConfig flavorsConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("host-large", 6., 6., 6, Flavor.Type.BARE_METAL); b.addFlavor("host-small", 3., 3., 3, Flavor.Type.BARE_METAL); b.addFlavor("host-medium", 4., 4., 4, Flavor.Type.BARE_METAL); b.addFlavor("d-1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER); b.addFlavor("d-2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER); - if (includeHeadroom) { - b.addFlavor("d-2-4", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER, 4); - } b.addFlavor("d-3", 3, 3., 3, Flavor.Type.DOCKER_CONTAINER); b.addFlavor("d-3-disk", 3, 3., 5, Flavor.Type.DOCKER_CONTAINER); b.addFlavor("d-3-mem", 3, 5., 3, Flavor.Type.DOCKER_CONTAINER); @@ -492,10 +315,6 @@ public class DynamicDockerProvisioningTest { return b.build(); } - private FlavorsConfig flavorsConfig() { - return flavorsConfig(false); - } - private void deployZoneApp(ProvisioningTester tester) { ApplicationId applicationId = tester.makeApplicationId(); List<HostSpec> list = tester.prepare(applicationId, diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index 8ffe4807427..7af7951abba 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -192,8 +192,6 @@ public: virtual void replay(const SplitBucketOperation &op) override { print(op); } virtual void replay(const JoinBucketsOperation &op) override { print(op); } virtual void replay(const PruneRemovedDocumentsOperation &op) override { print(op); } - virtual void replay(const SpoolerReplayStartOperation &op) override { print(op); } - virtual void replay(const SpoolerReplayCompleteOperation &op) override { print(op); } virtual void replay(const MoveOperation &op) override { print(op); } virtual void replay(const CreateBucketOperation &op) override { print(op); } virtual void replay(const CompactLidSpaceOperation &op) override { print(op); } @@ -275,8 +273,6 @@ public: virtual void replay(const SplitBucketOperation &) override { } virtual void replay(const JoinBucketsOperation &) override { } virtual void replay(const PruneRemovedDocumentsOperation &) override { } - virtual void replay(const SpoolerReplayStartOperation &) override { } - virtual void replay(const SpoolerReplayCompleteOperation &) override { } virtual void replay(const MoveOperation &) override { } virtual void replay(const CreateBucketOperation &) override { } }; diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp index e87e9209a17..6a9dc42b56d 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp @@ -12,7 +12,6 @@ #include <vespa/searchcore/proton/feedoperation/putoperation.h> #include <vespa/searchcore/proton/feedoperation/removeoperation.h> #include <vespa/searchcore/proton/feedoperation/splitbucketoperation.h> -#include <vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h> #include <vespa/searchcore/proton/feedoperation/updateoperation.h> #include <vespa/searchcore/proton/feedoperation/wipehistoryoperation.h> #include <vespa/searchlib/query/base.h> @@ -212,17 +211,6 @@ TEST("require that toString() on derived classes are meaningful") "target2=BucketId(0x000000000000002c), serialNum=0)", SplitBucketOperation(bucket_id1, bucket_id2, bucket_id3) .toString()); - - EXPECT_EQUAL("SpoolerReplayStart(spoolerSerialNum=0, serialNum=0)", - SpoolerReplayStartOperation().toString()); - EXPECT_EQUAL("SpoolerReplayStart(spoolerSerialNum=20, serialNum=10)", - SpoolerReplayStartOperation(10, 20).toString()); - - EXPECT_EQUAL("SpoolerReplayComplete(spoolerSerialNum=0, serialNum=0)", - SpoolerReplayCompleteOperation().toString()); - EXPECT_EQUAL("SpoolerReplayComplete(spoolerSerialNum=2, serialNum=1)", - SpoolerReplayCompleteOperation(1, 2).toString()); - EXPECT_EQUAL("Update(NULL, BucketId(0x0000000000000000), timestamp=0, dbdId=(subDbId=0, lid=0), " "prevDbdId=(subDbId=0, lid=0), prevMarkedAsRemoved=false, prevTimestamp=0, serialNum=0)", UpdateOperation().toString()); diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp index 93153a920cf..38309284e54 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp @@ -22,8 +22,6 @@ GroupingContext::deserialize(const char *groupSpec, uint32_t groupSpecLen) for (size_t i = 0; i < numGroupings; i++) { GroupingPtr grouping(new search::aggregation::Grouping); grouping->deserialize(nis); - aggregation::Attribute2AttributeKeyed attr2AttrKeyed; - grouping->select(attr2AttrKeyed, attr2AttrKeyed); grouping->setClock(&_clock); grouping->setTimeOfDoom(_timeOfDoom); _groupingList.push_back(grouping); diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/feedoperation/CMakeLists.txt index d64fbc6722f..f5e09b81313 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/CMakeLists.txt @@ -16,7 +16,6 @@ vespa_add_library(searchcore_feedoperation STATIC removedocumentsoperation.cpp removeoperation.cpp splitbucketoperation.cpp - spoolerreplayoperation.cpp updateoperation.cpp wipehistoryoperation.cpp DEPENDS diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h index 77b95547bd0..3509af0de5c 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h @@ -33,8 +33,6 @@ public: SPLIT_BUCKET = 10, JOIN_BUCKETS = 11, PRUNE_REMOVED_DOCUMENTS = 12, - SPOOLER_REPLAY_START = 13, - SPOOLER_REPLAY_COMPLETE = 14, MOVE = 15, CREATE_BUCKET = 16, COMPACT_LID_SPACE = 17, diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/operations.h b/searchcore/src/vespa/searchcore/proton/feedoperation/operations.h index 2cdf92fc8b7..df9f22b2462 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/operations.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/operations.h @@ -15,7 +15,6 @@ #include "removedocumentsoperation.h" #include "removeoperation.h" #include "splitbucketoperation.h" -#include "spoolerreplayoperation.h" #include "updateoperation.h" #include "wipehistoryoperation.h" diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp deleted file mode 100644 index 16ddedc4745..00000000000 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "spoolerreplayoperation.h" -#include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/util/stringfmt.h> - -using vespalib::make_string; - -namespace proton { - - -SpoolerReplayOperation::SpoolerReplayOperation(Type type) - : FeedOperation(type), - _spoolerSerialNum() -{ -} - -SpoolerReplayOperation::SpoolerReplayOperation(Type type, SerialNum serialNum, SerialNum spoolerSerialNum) - : FeedOperation(type), - _spoolerSerialNum(spoolerSerialNum) -{ - setSerialNum(serialNum); -} - - -void -SpoolerReplayOperation::serialize(vespalib::nbostream &os) const -{ - os << _spoolerSerialNum; -} - - -void -SpoolerReplayOperation::deserialize(vespalib::nbostream &is) -{ - is >> _spoolerSerialNum; -} - -vespalib::string SpoolerReplayOperation::toString() const { - return make_string("SpoolerReplay%s(spoolerSerialNum=%" PRIu64", serialNum=%" PRIu64 ")", - getType() == SPOOLER_REPLAY_START ? "Start" : "Complete", _spoolerSerialNum, getSerialNum()); -} - - -SpoolerReplayStartOperation::SpoolerReplayStartOperation() - : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_START) -{ -} - - -SpoolerReplayStartOperation::SpoolerReplayStartOperation(SerialNum serialNum, SerialNum spoolerSerialNum) - : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_START, - serialNum, - spoolerSerialNum) -{ -} - - -SpoolerReplayCompleteOperation::SpoolerReplayCompleteOperation() - : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_COMPLETE) -{ -} - - -SpoolerReplayCompleteOperation::SpoolerReplayCompleteOperation(SerialNum serialNum, - SerialNum spoolerSerialNum) - : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_COMPLETE, serialNum, spoolerSerialNum) -{ -} - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h deleted file mode 100644 index 028ad1c6bfa..00000000000 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "feedoperation.h" - -namespace proton { - -class SpoolerReplayOperation : public FeedOperation -{ -private: - SerialNum _spoolerSerialNum; -protected: - SpoolerReplayOperation(Type type); - SpoolerReplayOperation(Type type, SerialNum serialNum, SerialNum spoolerSerialNum); -public: - ~SpoolerReplayOperation() override {} - SerialNum getSpoolerSerialNum() const { return _spoolerSerialNum; } - void serialize(vespalib::nbostream &os) const override; - void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &) override { - deserialize(is); - } - void deserialize(vespalib::nbostream &is); - virtual vespalib::string toString() const override; -}; - - -/** - * Indicate that we are starting replaying the spooler log. - */ -class SpoolerReplayStartOperation : public SpoolerReplayOperation -{ -public: - SpoolerReplayStartOperation(); - /** - * @param serialNum the current serial number of the transaction log. - * @param spoolerSerialNum the serial number of the first entry of the spooler log replay. - */ - SpoolerReplayStartOperation(SerialNum serialNum, SerialNum spoolerSerialNum); -}; - - -/** - * Indicate that we are complete replaying the spooler log. - */ -class SpoolerReplayCompleteOperation : public SpoolerReplayOperation -{ -public: - SpoolerReplayCompleteOperation(); - /** - * @param serialNum the current serial number of the transaction log. - * @param spoolerSerialNum the serial number of the last entry of the spooler log replay. - */ - SpoolerReplayCompleteOperation(SerialNum serialNum, SerialNum spoolerSerialNum); -}; - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp index 0c2870ad35d..edbf0c631a5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp @@ -84,7 +84,8 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) tune._index._indexing._write.setFromConfig<ProtonConfig::Indexing::Write>(conf.indexing.write.io); tune._index._indexing._read.setFromConfig<ProtonConfig::Indexing::Read>(conf.indexing.read.io); tune._attr._write.setFromConfig<ProtonConfig::Attribute::Write>(conf.attribute.write.io); - tune._index._search._read.setFromConfig<ProtonConfig::Search, ProtonConfig::Search::Mmap>(conf.search.io, conf.search.mmap); + tune._index._search._read.setWantMemoryMap(); + tune._index._search._read.setFromMmapConfig<ProtonConfig::Search::Mmap>(conf.search.mmap); tune._summary._write.setFromConfig<ProtonConfig::Summary::Write>(conf.summary.write.io); tune._summary._seqRead.setFromConfig<ProtonConfig::Summary::Read>(conf.summary.read.io); tune._summary._randRead.setFromConfig<ProtonConfig::Summary::Read, ProtonConfig::Summary::Read::Mmap>(conf.summary.read.io, conf.summary.read.mmap); diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp index ae323bc93df..e45e3d7e423 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp @@ -112,12 +112,6 @@ public: virtual void replay(const PruneRemovedDocumentsOperation &op) override { _feed_view_ptr->handlePruneRemovedDocuments(op); } - virtual void replay(const SpoolerReplayStartOperation &op) override { - (void) op; - } - virtual void replay(const SpoolerReplayCompleteOperation &op) override { - (void) op; - } virtual void replay(const MoveOperation &op) override { _feed_view_ptr->handleMove(op, search::IDestructorCallback::SP()); } diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h b/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h index e93821e2a36..9acc3a530d3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h @@ -16,8 +16,6 @@ class DeleteBucketOperation; class SplitBucketOperation; class JoinBucketsOperation; class PruneRemovedDocumentsOperation; -class SpoolerReplayStartOperation; -class SpoolerReplayCompleteOperation; class MoveOperation; class CreateBucketOperation; class CompactLidSpaceOperation; @@ -41,8 +39,6 @@ struct IReplayPacketHandler virtual void replay(const SplitBucketOperation &op) = 0; virtual void replay(const JoinBucketsOperation &op) = 0; virtual void replay(const PruneRemovedDocumentsOperation &op) = 0; - virtual void replay(const SpoolerReplayStartOperation &op) = 0; - virtual void replay(const SpoolerReplayCompleteOperation &op) = 0; virtual void replay(const MoveOperation &op) = 0; virtual void replay(const CreateBucketOperation &op) = 0; virtual void replay(const CompactLidSpaceOperation &op) = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index ab0abde6ccd..070a3172fe0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -14,7 +14,6 @@ #include "searchhandlerproxy.h" #include "simpleflush.h" -#include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchcore/proton/flushengine/flushengine.h> #include <vespa/searchcore/proton/flushengine/flush_engine_explorer.h> #include <vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h> @@ -48,9 +47,6 @@ using vespalib::Slime; using vespalib::slime::ArrayInserter; using vespalib::slime::Cursor; -using search::TuneFileDocumentDB; -using search::index::Schema; -using search::index::SchemaBuilder; using search::transactionlog::DomainStats; using vespa::config::search::core::ProtonConfig; using vespa::config::search::core::internal::InternalProtonType; diff --git a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp index 42451f08315..447bd0d8624 100644 --- a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp @@ -74,14 +74,6 @@ ReplayPacketDispatcher::replayEntry(const Packet::Entry &entry) PruneRemovedDocumentsOperation op; replay(op, is, entry); break; - } case FeedOperation::SPOOLER_REPLAY_START: { - SpoolerReplayStartOperation op; - replay(op, is, entry); - break; - } case FeedOperation::SPOOLER_REPLAY_COMPLETE: { - SpoolerReplayCompleteOperation op; - replay(op, is, entry); - break; } case FeedOperation::MOVE: { MoveOperation op; replay(op, is, entry); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index c6d8f70fde8..da34ab8822d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -3,13 +3,16 @@ package com.yahoo.searchlib.rankingexpression; import com.google.common.collect.ImmutableList; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.text.Utf8; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.*; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * A function defined by a ranking expression @@ -24,6 +27,16 @@ public class ExpressionFunction { private final RankingExpression body; /** + * Constructs a new function with no arguments + * + * @param name the name of this function + * @param body the ranking expression that defines this function + */ + public ExpressionFunction(String name, RankingExpression body) { + this(name, Collections.emptyList(), body); + } + + /** * Constructs a new function * * @param name the name of this function diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index ed82ba20fbe..722520fea08 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -250,12 +250,12 @@ public class RankingExpression implements Serializable { /** * Creates the necessary rank properties required to implement this expression. * - * @param macros the expression macros to expand. - * @return a list of named rank properties required to implement this expression. + * @param functions the expression functions to expand + * @return a list of named rank properties required to implement this expression */ - public Map<String, String> getRankProperties(List<ExpressionFunction> macros) { + public Map<String, String> getRankProperties(List<ExpressionFunction> functions) { Deque<String> path = new LinkedList<>(); - SerializationContext context = new SerializationContext(macros); + SerializationContext context = new SerializationContext(functions); String serializedRoot = root.toString(new StringBuilder(), context, path, null).toString(); Map<String, String> serializedExpressions = context.serializedFunctions(); serializedExpressions.put(propertyName(name), serializedRoot); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java index ac5eefcc5b2..282a4c5e0a9 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java @@ -30,8 +30,8 @@ public class ImportedModel { private final Map<String, Tensor> smallConstants = new HashMap<>(); private final Map<String, Tensor> largeConstants = new HashMap<>(); private final Map<String, RankingExpression> expressions = new HashMap<>(); - private final Map<String, RankingExpression> macros = new HashMap<>(); - private final Map<String, TensorType> requiredMacros = new HashMap<>(); + private final Map<String, RankingExpression> functions = new HashMap<>(); + private final Map<String, TensorType> requiredFunctions = new HashMap<>(); /** * Creates a new imported model. @@ -77,13 +77,13 @@ public class ImportedModel { public Map<String, RankingExpression> expressions() { return Collections.unmodifiableMap(expressions); } /** - * Returns an immutable map of macros that are part of this model. - * Note that the macros themselves are *not* copies and *not* immutable - they must be copied before modification. + * Returns an immutable map of the functions that are part of this model. + * Note that the functions themselves are *not* copies and *not* immutable - they must be copied before modification. */ - public Map<String, RankingExpression> macros() { return Collections.unmodifiableMap(macros); } + public Map<String, RankingExpression> functions() { return Collections.unmodifiableMap(functions); } - /** Returns an immutable map of the macros that must be provided by the environment running this model */ - public Map<String, TensorType> requiredMacros() { return Collections.unmodifiableMap(requiredMacros); } + /** Returns an immutable map of the functions that must be provided by the environment running this model */ + public Map<String, TensorType> requiredFunctions() { return Collections.unmodifiableMap(requiredFunctions); } /** Returns an immutable map of the signatures of this */ public Map<String, Signature> signatures() { return Collections.unmodifiableMap(signatures); } @@ -100,8 +100,8 @@ public class ImportedModel { void smallConstant(String name, Tensor constant) { smallConstants.put(name, constant); } void largeConstant(String name, Tensor constant) { largeConstants.put(name, constant); } void expression(String name, RankingExpression expression) { expressions.put(name, expression); } - void macro(String name, RankingExpression expression) { macros.put(name, expression); } - void requiredMacro(String name, TensorType type) { requiredMacros.put(name, type); } + void function(String name, RankingExpression expression) { functions.put(name, expression); } + void requiredFunction(String name, TensorType type) { requiredFunctions.put(name, type); } /** * Returns all the output expressions of this indexed by name. The names consist of one or two parts diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java index 2ae107a5770..d25502fd149 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java @@ -24,7 +24,7 @@ import java.util.logging.Logger; * ranking expressions. The general mechanism for import is for the * specific ML platform import implementations to create an * IntermediateGraph. This class offers common code to convert the - * IntermediateGraph to Vespa ranking expressions and macros. + * IntermediateGraph to Vespa ranking expressions and functions. * * @author lesters */ @@ -122,7 +122,7 @@ public abstract class ModelImporter { importExpressionInputs(operation, model); importRankingExpression(operation, model); importArgumentExpression(operation, model); - importMacroExpression(operation, model); + importFunctionExpression(operation, model); return operation.function(); } @@ -188,15 +188,15 @@ public abstract class ModelImporter { // All inputs must have dimensions with standard naming convention: d0, d1, ... OrderedTensorType standardNamingConvention = OrderedTensorType.standardType(operation.type().get()); model.argument(operation.vespaName(), standardNamingConvention.type()); - model.requiredMacro(operation.vespaName(), standardNamingConvention.type()); + model.requiredFunction(operation.vespaName(), standardNamingConvention.type()); } } - private static void importMacroExpression(IntermediateOperation operation, ImportedModel model) { - if (operation.macro().isPresent()) { - TensorFunction function = operation.macro().get(); + private static void importFunctionExpression(IntermediateOperation operation, ImportedModel model) { + if (operation.rankingExpressionFunction().isPresent()) { + TensorFunction function = operation.rankingExpressionFunction().get(); try { - model.macro(operation.macroName(), new RankingExpression(operation.macroName(), function.toString())); + model.function(operation.rankingExpressionFunctionName(), new RankingExpression(operation.rankingExpressionFunctionName(), function.toString())); } catch (ParseException e) { throw new RuntimeException("Tensorflow function " + function + diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java index 43de29cedd5..34f5f1365a1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java @@ -29,7 +29,7 @@ import java.util.function.Function; */ public abstract class IntermediateOperation { - private final static String MACRO_PREFIX = "imported_ml_macro_"; + private final static String FUNCTION_PREFIX = "imported_ml_function_"; protected final String name; protected final String modelName; @@ -38,7 +38,7 @@ public abstract class IntermediateOperation { protected OrderedTensorType type; protected TensorFunction function; - protected TensorFunction macro = null; + protected TensorFunction rankingExpressionFunction = null; private final List<String> importWarnings = new ArrayList<>(); private Value constantValue = null; @@ -71,8 +71,8 @@ public abstract class IntermediateOperation { ExpressionNode constant = new ReferenceNode(Reference.simple("constant", vespaName())); function = new TensorFunctionNode.TensorFunctionExpressionNode(constant); } else if (outputs.size() > 1) { - macro = lazyGetFunction(); - function = new VariableTensor(macroName(), type.type()); + rankingExpressionFunction = lazyGetFunction(); + function = new VariableTensor(rankingExpressionFunctionName(), type.type()); } else { function = lazyGetFunction(); } @@ -86,11 +86,13 @@ public abstract class IntermediateOperation { /** Return unmodifiable list of inputs */ public List<IntermediateOperation> inputs() { return inputs; } - /** Return unmodifiable list of outputs. If a node has multiple outputs, consider adding a macro. */ + /** Return unmodifiable list of outputs. If a node has multiple outputs, consider adding a function. */ public List<IntermediateOperation> outputs() { return Collections.unmodifiableList(outputs); } - /** Returns a Vespa ranking expression that should be added as a macro */ - public Optional<TensorFunction> macro() { return Optional.ofNullable(macro); } + /** Returns a function that should be added as a ranking expression function */ + public Optional<TensorFunction> rankingExpressionFunction() { + return Optional.ofNullable(rankingExpressionFunction); + } /** Add dimension name constraints for this operation */ public void addDimensionNameConstraints(DimensionRenamer renamer) { } @@ -131,8 +133,10 @@ public abstract class IntermediateOperation { public String vespaName() { return vespaName(name); } public String vespaName(String name) { return name != null ? namePartOf(name).replace('/', '_') : null; } - /** Retrieve the valid Vespa name of this node if it is a macro */ - public String macroName() { return vespaName() != null ? MACRO_PREFIX + modelName + "_" + vespaName() : null; } + /** Retrieve the valid Vespa name of this node if it is a ranking expression function */ + public String rankingExpressionFunctionName() { + return vespaName() != null ? FUNCTION_PREFIX + modelName + "_" + vespaName() : null; + } /** Retrieve the list of warnings produced during its lifetime */ public List<String> warnings() { return Collections.unmodifiableList(importWarnings); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java index 9299ae9be12..b335fd7e1c5 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java @@ -26,13 +26,13 @@ public class PlaceholderWithDefault extends IntermediateOperation { if (!allInputFunctionsPresent(1)) { return null; } - // This should be a call to the macro we add below, but for now + // This should be a call to the function we add below, but for now // we treat this as as identity function and just pass the constant. return inputs.get(0).function().orElse(null); } @Override - public Optional<TensorFunction> macro() { + public Optional<TensorFunction> rankingExpressionFunction() { // For now, it is much more efficient to assume we always will return // the default value, as we can prune away large parts of the expression // tree by having it calculated as a constant. If a case arises where @@ -42,7 +42,7 @@ public class PlaceholderWithDefault extends IntermediateOperation { @Override public boolean isConstant() { - return true; // not true if we add to macro + return true; // not true if we add to function } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index e2dc170c168..eb8d2229a6d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -15,7 +15,7 @@ import java.util.Deque; import java.util.List; /** - * A node referring either to a value in the context or to a named ranking expression (function aka macro). + * A node referring either to a value in the context or to a named ranking expression function. * * @author bratseth */ @@ -64,7 +64,7 @@ public final class ReferenceNode extends CompositeNode { @Override public StringBuilder toString(StringBuilder string, SerializationContext context, Deque<String> path, CompositeNode parent) { - // A reference to a macro argument? + // A reference to a function argument? if (reference.isIdentifier() && context.getBinding(getName()) != null) { // a bound identifier: replace by the value it is bound to return string.append(context.getBinding(getName())); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java index 7c929ae24b3..571e1f4d608 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java @@ -91,26 +91,26 @@ public class RankingExpressionTestCase { @Test public void testSelfRecursionSerialization() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("foo", null, new RankingExpression("foo"))); + List<ExpressionFunction> functions = new ArrayList<>(); + functions.add(new ExpressionFunction("foo", null, new RankingExpression("foo"))); RankingExpression exp = new RankingExpression("foo"); try { - exp.getRankProperties(macros); + exp.getRankProperties(functions); } catch (RuntimeException e) { assertEquals("Cycle in ranking expression function: [foo[]]", e.getMessage()); } } @Test - public void testMacroCycleSerialization() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("foo", null, new RankingExpression("bar"))); - macros.add(new ExpressionFunction("bar", null, new RankingExpression("foo"))); + public void testFunctionCycleSerialization() throws ParseException { + List<ExpressionFunction> funnctions = new ArrayList<>(); + funnctions.add(new ExpressionFunction("foo", null, new RankingExpression("bar"))); + funnctions.add(new ExpressionFunction("bar", null, new RankingExpression("foo"))); RankingExpression exp = new RankingExpression("foo"); try { - exp.getRankProperties(macros); + exp.getRankProperties(funnctions); } catch (RuntimeException e) { assertEquals("Cycle in ranking expression function: [foo[], bar[]]", e.getMessage()); } @@ -118,11 +118,11 @@ public class RankingExpressionTestCase { @Test public void testSerialization() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("foo", Arrays.asList("arg1", "arg2"), new RankingExpression("min(arg1, pow(arg2, 2))"))); - macros.add(new ExpressionFunction("bar", Arrays.asList("arg1", "arg2"), new RankingExpression("arg1 * arg1 + 2 * arg1 * arg2 + arg2 * arg2"))); - macros.add(new ExpressionFunction("baz", Arrays.asList("arg1", "arg2"), new RankingExpression("foo(1, 2) / bar(arg1, arg2)"))); - macros.add(new ExpressionFunction("cox", null, new RankingExpression("10 + 08 * 1977"))); + List<ExpressionFunction> functions = new ArrayList<>(); + functions.add(new ExpressionFunction("foo", Arrays.asList("arg1", "arg2"), new RankingExpression("min(arg1, pow(arg2, 2))"))); + functions.add(new ExpressionFunction("bar", Arrays.asList("arg1", "arg2"), new RankingExpression("arg1 * arg1 + 2 * arg1 * arg2 + arg2 * arg2"))); + functions.add(new ExpressionFunction("baz", Arrays.asList("arg1", "arg2"), new RankingExpression("foo(1, 2) / bar(arg1, arg2)"))); + functions.add(new ExpressionFunction("cox", null, new RankingExpression("10 + 08 * 1977"))); assertSerialization(Arrays.asList( "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) + rankingExpression(foo@af74e3fd9070bd18.a368ed0a5ba3a5d0) * rankingExpression(foo@dbab346efdad5362.e5c39e42ebd91c30)", @@ -130,19 +130,19 @@ public class RankingExpressionTestCase { "min(6,pow(7,2))", "min(1,pow(2,2))", "min(3,pow(4,2))", - "min(rankingExpression(foo@84951be88255b0ec.d0303e061b36fab8),pow(8,2))"), "foo(1,2) + foo(3,4) * foo(5, foo(foo(6, 7), 8))", macros); + "min(rankingExpression(foo@84951be88255b0ec.d0303e061b36fab8),pow(8,2))"), "foo(1,2) + foo(3,4) * foo(5, foo(foo(6, 7), 8))", functions); assertSerialization(Arrays.asList( "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) + rankingExpression(bar@af74e3fd9070bd18.a368ed0a5ba3a5d0)", "min(1,pow(2,2))", - "3 * 3 + 2 * 3 * 4 + 4 * 4"), "foo(1, 2) + bar(3, 4)", macros); + "3 * 3 + 2 * 3 * 4 + 4 * 4"), "foo(1, 2) + bar(3, 4)", functions); assertSerialization(Arrays.asList( "rankingExpression(baz@e2dc17a89864aed0.12232eb692c6c502)", "min(1,pow(2,2))", "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) / rankingExpression(bar@e2dc17a89864aed0.12232eb692c6c502)", - "1 * 1 + 2 * 1 * 2 + 2 * 2"), "baz(1, 2)", macros); + "1 * 1 + 2 * 1 * 2 + 2 * 2"), "baz(1, 2)", functions); assertSerialization(Arrays.asList( "rankingExpression(cox)", - "10 + 08 * 1977"), "cox", macros + "10 + 08 * 1977"), "cox", functions ); } @@ -159,8 +159,8 @@ public class RankingExpressionTestCase { @Test public void testBug3464208() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("log10tweetage", null, new RankingExpression("69"))); + List<ExpressionFunction> functions = new ArrayList<>(); + functions.add(new ExpressionFunction("log10tweetage", null, new RankingExpression("69"))); String lhs = "log10(0.01+attribute(user_followers_count)) * log10(socialratio) * " + "log10(userage/(0.01+attribute(user_statuses_count)))"; @@ -172,8 +172,8 @@ public class RankingExpressionTestCase { String expRhs = "(rankingExpression(log10tweetage) * rankingExpression(log10tweetage) * " + "rankingExpression(log10tweetage)) + 5.0 * attribute(ythl)"; - assertSerialization(Arrays.asList(expLhs + " + " + expRhs, "69"), lhs + " + " + rhs, macros); - assertSerialization(Arrays.asList(expLhs + " - " + expRhs, "69"), lhs + " - " + rhs, macros); + assertSerialization(Arrays.asList(expLhs + " + " + expRhs, "69"), lhs + " + " + rhs, functions); + assertSerialization(Arrays.asList(expLhs + " - " + expRhs, "69"), lhs + " - " + rhs, functions); } @Test @@ -295,12 +295,12 @@ public class RankingExpressionTestCase { assertEquals(expected, new RankingExpression(expression).toString()); } - /** Test serialization with no macros */ + /** Test serialization with no functions */ private void assertSerialization(String expectedSerialization, String expressionString) { String serializedExpression; try { RankingExpression expression = new RankingExpression(expressionString); - // No macros -> expect one rank property + // No functions -> expect one rank property serializedExpression = expression.getRankProperties(Collections.emptyList()).values().iterator().next(); assertEquals(expectedSerialization, serializedExpression); } @@ -309,7 +309,7 @@ public class RankingExpressionTestCase { } try { - // No macros -> output should be parseable to a ranking expression + // No functions -> output should be parseable to a ranking expression // (but not the same one due to primitivization) RankingExpression reparsedExpression = new RankingExpression(serializedExpression); // Serializing the primitivized expression should yield the same expression again @@ -323,17 +323,17 @@ public class RankingExpressionTestCase { } private void assertSerialization(List<String> expectedSerialization, String expressionString, - List<ExpressionFunction> macros) { - assertSerialization(expectedSerialization, expressionString, macros, false); + List<ExpressionFunction> functions) { + assertSerialization(expectedSerialization, expressionString, functions, false); } private void assertSerialization(List<String> expectedSerialization, String expressionString, - List<ExpressionFunction> macros, boolean print) { + List<ExpressionFunction> functions, boolean print) { try { if (print) System.out.println("Parsing expression '" + expressionString + "'."); RankingExpression expression = new RankingExpression(expressionString); - Map<String, String> rankProperties = expression.getRankProperties(macros); + Map<String, String> rankProperties = expression.getRankProperties(functions); if (print) { for (String key : rankProperties.keySet()) System.out.println("Property '" + key + "': " + rankProperties.get(key)); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java index a63c7346335..a8f7542f3a4 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java @@ -18,11 +18,11 @@ public class DropoutImportTestCase { public void testDropoutImport() { TestableTensorFlowModel model = new TestableTensorFlowModel("test", "src/test/files/integration/tensorflow/dropout/saved"); - // Check required macros - assertEquals(1, model.get().requiredMacros().size()); - assertTrue(model.get().requiredMacros().containsKey("X")); + // Check required functions + assertEquals(1, model.get().requiredFunctions().size()); + assertTrue(model.get().requiredFunctions().containsKey("X")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.get().requiredMacros().get("X")); + model.get().requiredFunctions().get("X")); ImportedModel.Signature signature = model.get().signature("serving_default"); @@ -32,7 +32,7 @@ public class DropoutImportTestCase { RankingExpression output = signature.outputExpression("y"); assertNotNull(output); assertEquals("outputs/Maximum", output.getName()); - assertEquals("join(join(imported_ml_macro_test_outputs_BiasAdd, reduce(constant(test_outputs_Const), sum, d1), f(a,b)(a * b)), imported_ml_macro_test_outputs_BiasAdd, f(a,b)(max(a,b)))", + assertEquals("join(join(imported_ml_function_test_outputs_BiasAdd, reduce(constant(test_outputs_Const), sum, d1), f(a,b)(a * b)), imported_ml_function_test_outputs_BiasAdd, f(a,b)(max(a,b)))", output.getRoot().toString()); model.assertEqualResult("X", output.getName()); } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java index bcfc6ce0a04..e20ac16a691 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java @@ -36,11 +36,11 @@ public class OnnxMnistSoftmaxImportTestCase { constant1.type()); assertEquals(10, constant1.size()); - // Check required macros (inputs) - assertEquals(1, model.requiredMacros().size()); - assertTrue(model.requiredMacros().containsKey("Placeholder")); + // Check required functions (inputs) + assertEquals(1, model.requiredFunctions().size()); + assertTrue(model.requiredFunctions().containsKey("Placeholder")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.requiredMacros().get("Placeholder")); + model.requiredFunctions().get("Placeholder")); // Check outputs RankingExpression output = model.defaultSignature().outputExpression("add"); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java index dd6c8095e3c..ef28eb4678f 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java @@ -34,14 +34,14 @@ public class TensorFlowMnistSoftmaxImportTestCase { constant1.type()); assertEquals(10, constant1.size()); - // Check (provided) macros - assertEquals(0, model.get().macros().size()); + // Check (provided) functions + assertEquals(0, model.get().functions().size()); - // Check required macros - assertEquals(1, model.get().requiredMacros().size()); - assertTrue(model.get().requiredMacros().containsKey("Placeholder")); + // Check required functions + assertEquals(1, model.get().requiredFunctions().size()); + assertTrue(model.get().requiredFunctions().containsKey("Placeholder")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.get().requiredMacros().get("Placeholder")); + model.get().requiredFunctions().get("Placeholder")); // Check signatures assertEquals(1, model.get().signatures().size()); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java index 4de3aa5d635..5447e5240f7 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java @@ -48,7 +48,7 @@ public class TestableTensorFlowModel { Tensor placeholder = placeholderArgument(); context.put(inputName, new TensorValue(placeholder)); - model.macros().forEach((k,v) -> evaluateMacro(context, model, k)); + model.functions().forEach((k, v) -> evaluateFunction(context, model, k)); Tensor vespaResult = model.expressions().get(operationName).evaluate(context).asTensor(); assertEquals("Operation '" + operationName + "' produces equal results", @@ -62,7 +62,7 @@ public class TestableTensorFlowModel { Tensor placeholder = placeholderArgument(); context.put(inputName, new TensorValue(placeholder)); - model.macros().forEach((k,v) -> evaluateMacro(context, model, k)); + model.functions().forEach((k, v) -> evaluateFunction(context, model, k)); Tensor vespaResult = model.expressions().get(operationName).evaluate(context).asTensor(); assertEquals("Operation '" + operationName + "' produces equal results", tfResult, vespaResult); @@ -96,24 +96,24 @@ public class TestableTensorFlowModel { return b.build(); } - private void evaluateMacro(Context context, ImportedModel model, String macroName) { - if (!context.names().contains(macroName)) { - RankingExpression e = model.macros().get(macroName); - evaluateMacroDependencies(context, model, e.getRoot()); - context.put(macroName, new TensorValue(e.evaluate(context).asTensor())); + private void evaluateFunction(Context context, ImportedModel model, String functionName) { + if (!context.names().contains(functionName)) { + RankingExpression e = model.functions().get(functionName); + evaluateFunctionDependencies(context, model, e.getRoot()); + context.put(functionName, new TensorValue(e.evaluate(context).asTensor())); } } - private void evaluateMacroDependencies(Context context, ImportedModel model, ExpressionNode node) { + private void evaluateFunctionDependencies(Context context, ImportedModel model, ExpressionNode node) { if (node instanceof ReferenceNode) { String name = node.toString(); - if (model.macros().containsKey(name)) { - evaluateMacro(context, model, name); + if (model.functions().containsKey(name)) { + evaluateFunction(context, model, name); } } else if (node instanceof CompositeNode) { for (ExpressionNode child : ((CompositeNode)node).children()) { - evaluateMacroDependencies(context, model, child); + evaluateFunctionDependencies(context, model, child); } } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java index 84e51835458..1f28f0b0129 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java @@ -27,7 +27,7 @@ public class ConstantDereferencerTestCase { TransformContext context = new TransformContext(constants); assertEquals("1.0 + 2.0 + 3.5", c.transform(new RankingExpression("a + b + c"), context).toString()); - assertEquals("myMacro(1.0,2.0)", c.transform(new RankingExpression("myMacro(a, b)"), context).toString()); + assertEquals("myFunction(1.0,2.0)", c.transform(new RankingExpression("myFunction(a, b)"), context).toString()); } } diff --git a/searchlib/src/tests/expression/attributenode/CMakeLists.txt b/searchlib/src/tests/expression/attributenode/CMakeLists.txt index c7df5458bb7..3006c27dd0d 100644 --- a/searchlib/src/tests/expression/attributenode/CMakeLists.txt +++ b/searchlib/src/tests/expression/attributenode/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchlib_attribute_node_test_app TEST attribute_node_test.cpp DEPENDS searchlib + searchlib_test ) vespa_add_test(NAME searchlib_attribute_node_test_app COMMAND searchlib_attribute_node_test_app) diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp index c92b5fc4808..7490b0699be 100644 --- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp +++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp @@ -10,8 +10,8 @@ #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/stringbase.h> #include <vespa/searchlib/expression/attributenode.h> -#include <vespa/searchlib/expression/attribute_keyed_node.h> #include <vespa/searchlib/expression/resultvector.h> +#include <vespa/searchlib/test/make_attribute_map_lookup_node.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/testapp.h> @@ -31,7 +31,6 @@ using search::attribute::Config; using search::attribute::IAttributeVector; using search::attribute::getUndefined; using search::expression::AttributeNode; -using search::expression::AttributeKeyedNode; using search::expression::EnumResultNode; using search::expression::EnumResultNodeVector; using search::expression::FloatResultNode; @@ -44,6 +43,7 @@ using search::expression::ResultNode; using search::expression::ResultNodeVector; using search::expression::StringResultNode; using search::expression::StringResultNodeVector; +using search::expression::test::makeAttributeMapLookupNode; using vespalib::BufferRef; namespace { @@ -220,7 +220,7 @@ Fixture::makeNode(const vespalib::string &attributeName, bool useEnumOptimizatio if (attributeName.find('{') == vespalib::string::npos) { node = std::make_unique<AttributeNode>(attributeName); } else { - node = std::make_unique<AttributeKeyedNode>(attributeName); + node = makeAttributeMapLookupNode(attributeName); } if (useEnumOptimization) { node->useEnumOptimization(); diff --git a/searchlib/src/tests/grouping/CMakeLists.txt b/searchlib/src/tests/grouping/CMakeLists.txt index ea90f27d9f6..e2bb73c57a4 100644 --- a/searchlib/src/tests/grouping/CMakeLists.txt +++ b/searchlib/src/tests/grouping/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_executable(searchlib_grouping_test_app TEST grouping_test.cpp DEPENDS searchlib + searchlib_test ) vespa_add_test(NAME searchlib_grouping_test_app COMMAND searchlib_grouping_test_app) vespa_add_executable(searchlib_hyperloglog_test_app TEST diff --git a/searchlib/src/tests/grouping/grouping_test.cpp b/searchlib/src/tests/grouping/grouping_test.cpp index 4cf9eb6f5c6..084f13795f7 100644 --- a/searchlib/src/tests/grouping/grouping_test.cpp +++ b/searchlib/src/tests/grouping/grouping_test.cpp @@ -9,6 +9,8 @@ #include <vespa/searchlib/aggregation/fs4hit.h> #include <vespa/searchlib/aggregation/predicates.h> #include <vespa/searchlib/expression/fixedwidthbucketfunctionnode.h> +#include <vespa/searchlib/test/make_attribute_map_lookup_node.h> +#include <vespa/searchcommon/common/undefinedvalues.h> #include <algorithm> #include <cmath> #include <iostream> @@ -21,6 +23,13 @@ using namespace search; using namespace search::aggregation; using namespace search::attribute; using namespace search::expression; +using search::expression::test::makeAttributeMapLookupNode; + +namespace { + +const int64_t undefinedInteger = getUndefined<int64_t>(); + +} //----------------------------------------------------------------------------- @@ -61,6 +70,14 @@ public: _attr->add(value); return *this; } + AttrBuilder &add(std::vector<T> values) { + DocId ignore; + _attr->addDoc(ignore); + for (T value : values) { + _attr->add(value); + } + return *this; + } AttributeVector::SP sp() const { return _attrSP; } @@ -70,6 +87,9 @@ typedef AttrBuilder<SingleIntegerExtAttribute, int64_t> IntAttrBuilder; typedef AttrBuilder<SingleFloatExtAttribute, double> FloatAttrBuilder; typedef AttrBuilder<SingleStringExtAttribute, const char *> StringAttrBuilder; +using StringArrayAttrBuilder = AttrBuilder<MultiStringExtAttribute, const char *>; +using IntArrayAttrBuilder = AttrBuilder<MultiIntegerExtAttribute, int64_t>; + //----------------------------------------------------------------------------- class ResultBuilder @@ -164,8 +184,10 @@ public: void testFixedWidthBuckets(); void testThatNanIsConverted(); void testNanSorting(); + void testAttributeMapLookup(); int Main() override; private: + void testAggregationSimple(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const vespalib::string &name); void testAggregationSimpleSum(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const ResultNode & fr, const ResultNode & sr); class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate { @@ -313,6 +335,18 @@ prepareAggr(const AggregationResult & aggr, ExpressionNode::UP expr, const Resul prepared->setResult(r); return prepared; } + +void Test::testAggregationSimple(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const vespalib::string &name) +{ + ExpressionNode::CP clone(aggr); + Grouping request; + request.setRoot(Group().addResult(prepareAggr(aggr, makeAttributeMapLookupNode(name)))); + + Group expect; + expect.addResult(prepareAggr(aggr, makeAttributeMapLookupNode(name), ir)); + EXPECT_TRUE(testAggregation(ctx, request, expect)); +} + void Test::testAggregationSimpleSum(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const ResultNode & fr, const ResultNode & sr) { ExpressionNode::CP clone(aggr); @@ -1884,6 +1918,28 @@ Test::testThatNanIsConverted() ASSERT_EQUAL(g.getRank(), g.getRank()); } +void +Test::testAttributeMapLookup() +{ + AggregationContext ctx; + ctx.result().add(0).add(1); + ctx.add(StringArrayAttrBuilder("smap.key").add({"k1", "k2"}).add({"k3", "k4"}).sp()); + ctx.add(IntArrayAttrBuilder("smap.value.weight").add({10, 20}).add({100, 200}).sp()); + ctx.add(StringAttrBuilder("key1").add("k1").add("k4").sp()); + ctx.add(StringAttrBuilder("key2").add("k2").add("k3").sp()); + ctx.add(StringAttrBuilder("key3").add("k3").add("k2").sp()); + testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(10 + undefinedInteger), "smap{\"k1\"}.weight"); + testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(20 + undefinedInteger), "smap{\"k2\"}.weight"); + testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(2 * undefinedInteger), "smap{\"k5\"}.weight"); + testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(210), "smap{attribute(key1)}.weight"); + testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(120), "smap{attribute(key2)}.weight"); + testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(2 * undefinedInteger), "smap{attribute(key3)}.weight"); + testAggregationSimple(ctx, MinAggregationResult(), Int64ResultNode(10), "smap{attribute(key1)}.weight"); + testAggregationSimple(ctx, MinAggregationResult(), Int64ResultNode(20), "smap{attribute(key2)}.weight"); + testAggregationSimple(ctx, MaxAggregationResult(), Int64ResultNode(200), "smap{attribute(key1)}.weight"); + testAggregationSimple(ctx, MaxAggregationResult(), Int64ResultNode(100), "smap{attribute(key2)}.weight"); +} + //----------------------------------------------------------------------------- struct RunDiff { ~RunDiff() { system("diff -u lhs.out rhs.out > diff.txt"); }}; @@ -1916,6 +1972,7 @@ Test::Main() testTopN(); testThatNanIsConverted(); testNanSorting(); + testAttributeMapLookup(); TEST_DONE(); } diff --git a/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp b/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp index fe71484c4e6..5ffad122c7d 100644 --- a/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp @@ -4,7 +4,7 @@ #include "grouping.h" #include <vespa/searchlib/expression/multiargfunctionnode.h> #include <vespa/searchlib/expression/attributenode.h> -#include <vespa/searchlib/expression/attribute_keyed_node.h> +#include <vespa/searchlib/expression/attribute_map_lookup_node.h> #include <vespa/searchlib/expression/documentfieldnode.h> using namespace search::expression; @@ -64,18 +64,6 @@ Attribute2DocumentAccessor::getReplacementNode(const AttributeNode &attributeNod return std::make_unique<DocumentFieldNode>(attributeNode.getAttributeName()); } -std::unique_ptr<ExpressionNode> -Attribute2AttributeKeyed::getReplacementNode(const AttributeNode &attributeNode) -{ - const vespalib::string &attributeName = attributeNode.getAttributeName(); - auto lBracePos = attributeName.find('{'); - if (attributeNode.isKeyed() || lBracePos == vespalib::string::npos) { - return std::unique_ptr<ExpressionNode>(); - } else { - return std::make_unique<AttributeKeyedNode>(attributeName); - } -} - } // this function was added by ../../forcelink.sh diff --git a/searchlib/src/vespa/searchlib/aggregation/modifiers.h b/searchlib/src/vespa/searchlib/aggregation/modifiers.h index 6ffda313904..0120cb4eac9 100644 --- a/searchlib/src/vespa/searchlib/aggregation/modifiers.h +++ b/searchlib/src/vespa/searchlib/aggregation/modifiers.h @@ -28,10 +28,4 @@ private: std::unique_ptr<search::expression::ExpressionNode> getReplacementNode(const search::expression::AttributeNode &attributeNode) override; }; -class Attribute2AttributeKeyed : public AttributeNodeReplacer -{ -private: - std::unique_ptr<search::expression::ExpressionNode> getReplacementNode(const search::expression::AttributeNode &attributeNode) override; -}; - } diff --git a/searchlib/src/vespa/searchlib/common/identifiable.h b/searchlib/src/vespa/searchlib/common/identifiable.h index 5a64e29ddf3..35e49b5cddf 100644 --- a/searchlib/src/vespa/searchlib/common/identifiable.h +++ b/searchlib/src/vespa/searchlib/common/identifiable.h @@ -148,6 +148,7 @@ #define CID_search_expression_AggregationRefNode SEARCHLIB_CID(142) #define CID_search_expression_NormalizeSubjectFunctionNode SEARCHLIB_CID(143) #define CID_search_expression_DebugWaitFunctionNode SEARCHLIB_CID(144) +#define CID_search_expression_AttributeMapLookupNode SEARCHLIB_CID(145) #define CID_search_QueryNode SEARCHLIB_CID(150) #define CID_search_Query SEARCHLIB_CID(151) diff --git a/searchlib/src/vespa/searchlib/common/tunefileinfo.h b/searchlib/src/vespa/searchlib/common/tunefileinfo.h index 223dcdbc7f1..edd6d29dd58 100644 --- a/searchlib/src/vespa/searchlib/common/tunefileinfo.h +++ b/searchlib/src/vespa/searchlib/common/tunefileinfo.h @@ -119,8 +119,9 @@ public: int getAdvise() const { return _advise; } template <typename TuneControlConfig, typename MMapConfig> - void - setFromConfig(const enum TuneControlConfig::Io & tuneControlConfig, const MMapConfig & mmapFlags); + void setFromConfig(const enum TuneControlConfig::Io & tuneControlConfig, const MMapConfig & mmapFlags); + template <typename MMapConfig> + void setFromMmapConfig(const MMapConfig & mmapFlags); bool operator==(const TuneFileRandRead &rhs) const { return (_tuneControl == rhs._tuneControl) && (_mmapFlags == rhs._mmapFlags); diff --git a/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp b/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp index 22d89d904d2..08acd2caa97 100644 --- a/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp +++ b/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp @@ -1,3 +1,4 @@ + // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once @@ -17,6 +18,12 @@ TuneFileRandRead::setFromConfig(const enum TuneControlConfig::Io & tuneControlCo case TuneControlConfig::MMAP: _tuneControl = MMAP; break; default: _tuneControl = NORMAL; break; } + setFromMmapConfig(mmapFlags); +} + +template <typename MMapConfig> +void +TuneFileRandRead::setFromMmapConfig(const MMapConfig & mmapFlags) { for (size_t i(0), m(mmapFlags.options.size()); i < m; i++) { switch (mmapFlags.options[i]) { case MMapConfig::MLOCK: _mmapFlags |= MAP_LOCKED; break; diff --git a/searchlib/src/vespa/searchlib/expression/CMakeLists.txt b/searchlib/src/vespa/searchlib/expression/CMakeLists.txt index 944bc6f63df..652fa5a3b01 100644 --- a/searchlib/src/vespa/searchlib/expression/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/expression/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_expression OBJECT SOURCES - attribute_keyed_node.cpp + attribute_map_lookup_node.cpp attributenode.cpp attributeresult.cpp enumattributeresult.cpp diff --git a/searchlib/src/vespa/searchlib/expression/attribute_keyed_node.cpp b/searchlib/src/vespa/searchlib/expression/attribute_map_lookup_node.cpp index da6ed363b17..8a851b043aa 100644 --- a/searchlib/src/vespa/searchlib/expression/attribute_keyed_node.cpp +++ b/searchlib/src/vespa/searchlib/expression/attribute_map_lookup_node.cpp @@ -1,6 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "attribute_keyed_node.h" +#include "attribute_map_lookup_node.h" #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/searchcommon/attribute/attributecontent.h> @@ -11,11 +11,15 @@ using search::attribute::AttributeContent; using search::attribute::IAttributeVector; using search::attribute::BasicType; using search::attribute::getUndefined; +using vespalib::Deserializer; +using vespalib::Serializer; using EnumHandle = IAttributeVector::EnumHandle; namespace search::expression { -class AttributeKeyedNode::KeyHandler +IMPLEMENT_EXPRESSIONNODE(AttributeMapLookupNode, AttributeNode); + +class AttributeMapLookupNode::KeyHandler { protected: const IAttributeVector &_attribute; @@ -34,7 +38,7 @@ namespace { vespalib::string indirectKeyMarker("attribute("); -class BadKeyHandler : public AttributeKeyedNode::KeyHandler +class BadKeyHandler : public AttributeMapLookupNode::KeyHandler { public: BadKeyHandler(const IAttributeVector &attribute) @@ -70,7 +74,7 @@ EnumHandle convertKey<EnumHandle>(const IAttributeVector &attribute, const vespa } template <typename T, typename KeyType = T> -class KeyHandlerT : public AttributeKeyedNode::KeyHandler +class KeyHandlerT : public AttributeMapLookupNode::KeyHandler { AttributeContent<T> _keys; KeyType _key; @@ -119,7 +123,7 @@ matchingKey<const char *>(const char *lhs, const char *rhs) } template <typename T> -class IndirectKeyHandlerT : public AttributeKeyedNode::KeyHandler +class IndirectKeyHandlerT : public AttributeMapLookupNode::KeyHandler { const IAttributeVector &_keySourceAttribute; AttributeContent<T> _keys; @@ -157,9 +161,9 @@ using IndirectStringKeyHandler = IndirectKeyHandlerT<const char *>; class ValueHandler : public AttributeNode::Handler { protected: - std::unique_ptr<AttributeKeyedNode::KeyHandler> _keyHandler; + std::unique_ptr<AttributeMapLookupNode::KeyHandler> _keyHandler; const IAttributeVector &_attribute; - ValueHandler(std::unique_ptr<AttributeKeyedNode::KeyHandler> keyHandler, const IAttributeVector &attribute) + ValueHandler(std::unique_ptr<AttributeMapLookupNode::KeyHandler> keyHandler, const IAttributeVector &attribute) : _keyHandler(std::move(keyHandler)), _attribute(attribute) { @@ -173,7 +177,7 @@ class ValueHandlerT : public ValueHandler ResultNodeType &_result; T _undefinedValue; public: - ValueHandlerT(std::unique_ptr<AttributeKeyedNode::KeyHandler> keyHandler, const IAttributeVector &attribute, ResultNodeType &result, T undefinedValue) + ValueHandlerT(std::unique_ptr<AttributeMapLookupNode::KeyHandler> keyHandler, const IAttributeVector &attribute, ResultNodeType &result, T undefinedValue) : ValueHandler(std::move(keyHandler), attribute), _values(), _result(result), @@ -183,7 +187,7 @@ public: void handle(const AttributeResult & r) override { uint32_t docId = r.getDocId(); uint32_t keyIdx = _keyHandler->handle(docId); - if (keyIdx != AttributeKeyedNode::KeyHandler::noKeyIdx()) { + if (keyIdx != AttributeMapLookupNode::KeyHandler::noKeyIdx()) { _values.fill(_attribute, docId); if (keyIdx < _values.size()) { _result = _values[keyIdx]; @@ -228,7 +232,7 @@ IAttributeVector::largeint_t getUndefinedValue(BasicType::Type basicType) } -AttributeKeyedNode::AttributeKeyedNode() +AttributeMapLookupNode::AttributeMapLookupNode() : AttributeNode(), _keyAttributeName(), _valueAttributeName(), @@ -239,58 +243,35 @@ AttributeKeyedNode::AttributeKeyedNode() { } -AttributeKeyedNode::AttributeKeyedNode(const AttributeKeyedNode &) = default; +AttributeMapLookupNode::AttributeMapLookupNode(const AttributeMapLookupNode &) = default; -AttributeKeyedNode::AttributeKeyedNode(vespalib::stringref name) +AttributeMapLookupNode::AttributeMapLookupNode(vespalib::stringref name, vespalib::stringref keyAttributeName, vespalib::stringref valueAttributeName, vespalib::stringref key, vespalib::stringref keySourceAttributeName) : AttributeNode(name), - _keyAttributeName(), - _valueAttributeName(), - _key(), - _keySourceAttributeName(), + _keyAttributeName(keyAttributeName), + _valueAttributeName(valueAttributeName), + _key(key), + _keySourceAttributeName(keySourceAttributeName), _keyAttribute(nullptr), _keySourceAttribute(nullptr) { - setupAttributeNames(); } -AttributeKeyedNode::~AttributeKeyedNode() = default; +AttributeMapLookupNode::~AttributeMapLookupNode() = default; -AttributeKeyedNode & -AttributeKeyedNode::operator=(const AttributeKeyedNode &rhs) = default; - -void -AttributeKeyedNode::setupAttributeNames() -{ - vespalib::asciistream keyName; - vespalib::asciistream valueName; - auto leftBracePos = _attributeName.find('{'); - auto baseName = _attributeName.substr(0, leftBracePos); - auto rightBracePos = _attributeName.rfind('}'); - keyName << baseName << ".key"; - valueName << baseName << ".value" << _attributeName.substr(rightBracePos + 1); - _keyAttributeName = keyName.str(); - _valueAttributeName = valueName.str(); - if (rightBracePos != vespalib::string::npos && rightBracePos > leftBracePos) { - if (_attributeName[leftBracePos + 1] == '"' && _attributeName[rightBracePos - 1] == '"') { - _key = _attributeName.substr(leftBracePos + 2, rightBracePos - leftBracePos - 3); - } else if (_attributeName.substr(leftBracePos + 1, indirectKeyMarker.size()) == indirectKeyMarker && _attributeName[rightBracePos - 1] == ')') { - auto startPos = leftBracePos + 1 + indirectKeyMarker.size(); - _keySourceAttributeName = _attributeName.substr(startPos, rightBracePos - 1 - startPos); - } - } -} +AttributeMapLookupNode & +AttributeMapLookupNode::operator=(const AttributeMapLookupNode &rhs) = default; template <typename ResultNodeType> void -AttributeKeyedNode::prepareIntValues(std::unique_ptr<KeyHandler> keyHandler, const IAttributeVector &attribute, IAttributeVector::largeint_t undefinedValue) +AttributeMapLookupNode::prepareIntValues(std::unique_ptr<KeyHandler> keyHandler, const IAttributeVector &attribute, IAttributeVector::largeint_t undefinedValue) { auto resultNode = std::make_unique<ResultNodeType>(); _handler = std::make_unique<IntegerValueHandler<ResultNodeType>>(std::move(keyHandler), attribute, *resultNode, undefinedValue); setResultType(std::move(resultNode)); } -std::unique_ptr<AttributeKeyedNode::KeyHandler> -AttributeKeyedNode::makeKeyHandlerHelper() +std::unique_ptr<AttributeMapLookupNode::KeyHandler> +AttributeMapLookupNode::makeKeyHandlerHelper() { const IAttributeVector &attribute = *_keyAttribute; if (_keySourceAttribute != nullptr) { @@ -318,8 +299,8 @@ AttributeKeyedNode::makeKeyHandlerHelper() } } -std::unique_ptr<AttributeKeyedNode::KeyHandler> -AttributeKeyedNode::makeKeyHandler() +std::unique_ptr<AttributeMapLookupNode::KeyHandler> +AttributeMapLookupNode::makeKeyHandler() { try { return makeKeyHandlerHelper(); @@ -329,7 +310,7 @@ AttributeKeyedNode::makeKeyHandler() } void -AttributeKeyedNode::onPrepare(bool preserveAccurateTypes) +AttributeMapLookupNode::onPrepare(bool preserveAccurateTypes) { auto keyHandler = makeKeyHandler(); const IAttributeVector * attribute = _scratchResult->getAttribute(); @@ -380,7 +361,7 @@ AttributeKeyedNode::onPrepare(bool preserveAccurateTypes) } void -AttributeKeyedNode::cleanup() +AttributeMapLookupNode::cleanup() { _keyAttribute = nullptr; _keySourceAttribute = nullptr; @@ -388,7 +369,7 @@ AttributeKeyedNode::cleanup() } void -AttributeKeyedNode::wireAttributes(const search::attribute::IAttributeContext &attrCtx) +AttributeMapLookupNode::wireAttributes(const search::attribute::IAttributeContext &attrCtx) { auto valueAttribute = findAttribute(attrCtx, _useEnumOptimization, _valueAttributeName); _hasMultiValue = false; @@ -399,8 +380,20 @@ AttributeKeyedNode::wireAttributes(const search::attribute::IAttributeContext &a } } +Serializer & AttributeMapLookupNode::onSerialize(Serializer & os) const +{ + AttributeNode::onSerialize(os); + return os << _keyAttributeName << _valueAttributeName << _key << _keySourceAttributeName; +} + +Deserializer & AttributeMapLookupNode::onDeserialize(Deserializer & is) +{ + AttributeNode::onDeserialize(is); + return is >> _keyAttributeName >> _valueAttributeName >> _key >> _keySourceAttributeName; +} + void -AttributeKeyedNode::visitMembers(vespalib::ObjectVisitor &visitor) const +AttributeMapLookupNode::visitMembers(vespalib::ObjectVisitor &visitor) const { AttributeNode::visitMembers(visitor); visit(visitor, "keyAttributeName", _keyAttributeName); diff --git a/searchlib/src/vespa/searchlib/expression/attribute_keyed_node.h b/searchlib/src/vespa/searchlib/expression/attribute_map_lookup_node.h index e2cf8943aae..2f9c6328969 100644 --- a/searchlib/src/vespa/searchlib/expression/attribute_keyed_node.h +++ b/searchlib/src/vespa/searchlib/expression/attribute_map_lookup_node.h @@ -9,7 +9,7 @@ namespace search::expression { * Extract map value from attribute for the map key specified in the * grouping expression. */ -class AttributeKeyedNode : public AttributeNode +class AttributeMapLookupNode : public AttributeNode { public: using IAttributeVector = search::attribute::IAttributeVector; @@ -22,7 +22,6 @@ private: const IAttributeVector *_keyAttribute; const IAttributeVector *_keySourceAttribute; - void setupAttributeNames(); template <typename ResultNodeType> void prepareIntValues(std::unique_ptr<KeyHandler> keyHandler, const IAttributeVector &attribute, IAttributeVector::largeint_t undefinedValue); std::unique_ptr<KeyHandler> makeKeyHandlerHelper(); @@ -31,15 +30,16 @@ private: void wireAttributes(const search::attribute::IAttributeContext & attrCtx) override; void onPrepare(bool preserveAccurateTypes) override; public: - AttributeKeyedNode(); - AttributeKeyedNode(vespalib::stringref name); - AttributeKeyedNode(const AttributeKeyedNode &); - AttributeKeyedNode(AttributeKeyedNode &&) = delete; - ~AttributeKeyedNode() override; - AttributeKeyedNode &operator=(const AttributeKeyedNode &rhs); - AttributeKeyedNode &operator=(AttributeKeyedNode &&rhs) = delete; + DECLARE_NBO_SERIALIZE; + DECLARE_EXPRESSIONNODE(AttributeMapLookupNode); + AttributeMapLookupNode(); + AttributeMapLookupNode(vespalib::stringref name, vespalib::stringref keyAttributeName, vespalib::stringref valueAttributeName, vespalib::stringref key, vespalib::stringref keySourceAttributeName); + AttributeMapLookupNode(const AttributeMapLookupNode &); + AttributeMapLookupNode(AttributeMapLookupNode &&) = delete; + ~AttributeMapLookupNode() override; + AttributeMapLookupNode &operator=(const AttributeMapLookupNode &rhs); + AttributeMapLookupNode &operator=(AttributeMapLookupNode &&rhs) = delete; void visitMembers(vespalib::ObjectVisitor &visitor) const override; - bool isKeyed() const override { return true; } }; } diff --git a/searchlib/src/vespa/searchlib/expression/attributenode.h b/searchlib/src/vespa/searchlib/expression/attributenode.h index e12b5490955..472267f4b5c 100644 --- a/searchlib/src/vespa/searchlib/expression/attributenode.h +++ b/searchlib/src/vespa/searchlib/expression/attributenode.h @@ -55,7 +55,6 @@ public: void useEnumOptimization(bool use=true) { _useEnumOptimization = use; } bool hasMultiValue() const { return _hasMultiValue; } - virtual bool isKeyed() const { return false; } public: class Handler { diff --git a/searchlib/src/vespa/searchlib/test/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/CMakeLists.txt index b4fb895c0e2..1231a99920e 100644 --- a/searchlib/src/vespa/searchlib/test/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(searchlib_test SOURCES document_weight_attribute_helper.cpp initrange.cpp + make_attribute_map_lookup_node.cpp mock_attribute_context.cpp mock_attribute_manager.cpp searchiteratorverifier.cpp diff --git a/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.cpp b/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.cpp new file mode 100644 index 00000000000..6f717d99d2c --- /dev/null +++ b/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.cpp @@ -0,0 +1,38 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "make_attribute_map_lookup_node.h" +#include <vespa/searchlib/expression/attribute_map_lookup_node.h> +#include <vespa/vespalib/stllike/asciistream.h> + +namespace search::expression::test { + +namespace { + +vespalib::string indirectKeyMarker("attribute("); + +} + +std::unique_ptr<AttributeNode> +makeAttributeMapLookupNode(const vespalib::string attributeName) +{ + vespalib::asciistream keyName; + vespalib::asciistream valueName; + auto leftBracePos = attributeName.find('{'); + auto baseName = attributeName.substr(0, leftBracePos); + auto rightBracePos = attributeName.rfind('}'); + keyName << baseName << ".key"; + valueName << baseName << ".value" << attributeName.substr(rightBracePos + 1); + if (rightBracePos != vespalib::string::npos && rightBracePos > leftBracePos) { + if (attributeName[leftBracePos + 1] == '"' && attributeName[rightBracePos - 1] == '"') { + vespalib::string key = attributeName.substr(leftBracePos + 2, rightBracePos - leftBracePos - 3); + return std::make_unique<AttributeMapLookupNode>(attributeName, keyName.str(), valueName.str(), key, ""); + } else if (attributeName.substr(leftBracePos + 1, indirectKeyMarker.size()) == indirectKeyMarker && attributeName[rightBracePos - 1] == ')') { + auto startPos = leftBracePos + 1 + indirectKeyMarker.size(); + vespalib::string keySourceAttributeName = attributeName.substr(startPos, rightBracePos - 1 - startPos); + return std::make_unique<AttributeMapLookupNode>(attributeName, keyName.str(), valueName.str(), "", keySourceAttributeName); + } + } + return std::unique_ptr<AttributeNode>(); +} + +} diff --git a/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.h b/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.h new file mode 100644 index 00000000000..3434c8f2ae3 --- /dev/null +++ b/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.h @@ -0,0 +1,14 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <memory> + +namespace search::expression { class AttributeNode; } + +namespace search::expression::test { + +std::unique_ptr<AttributeNode> +makeAttributeMapLookupNode(const vespalib::string attributeName); + +} diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java index d88245587d2..8d2cd429517 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java @@ -20,6 +20,7 @@ import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo; import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.Zone; import com.yahoo.container.di.config.SubscriberFactory; @@ -209,17 +210,22 @@ public class StandaloneContainerApplication implements Application { } private static ContainerModelBuilder newContainerModelBuilder(Networking networkingOption) { + return isConfigServer() ? + new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables()) : + new ContainerModelBuilder(true, networkingOption); + } + + private static boolean isConfigServer() { Optional<String> profile = optionalInstallVariable(DEPLOYMENT_PROFILE_INSTALL_VARIABLE); if (profile.isPresent()) { String profileName = profile.get(); - if ("configserver".equals(profileName)) { - return new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables()); - } else { + if (profileName.equals("configserver")) + return true; + else throw new RuntimeException("Invalid deployment profile '" + profileName + "'"); - } - } else { - return new ContainerModelBuilder(true, networkingOption); } + + return false; } static Pair<VespaModel, Container> createContainerModel(Path applicationPath, FileRegistry fileRegistry, @@ -229,8 +235,7 @@ public class StandaloneContainerApplication implements Application { .includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build(); ApplicationPackage applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), logger); validateApplication(applicationPackage); - DeployState deployState = new DeployState.Builder().applicationPackage(applicationPackage).fileRegistry(fileRegistry) - .deployLogger(logger).configDefinitionRepo(configDefinitionRepo).build(); + DeployState deployState = createDeployState(applicationPackage, fileRegistry, logger); VespaModel root = VespaModel.createIncomplete(deployState); ApplicationConfigProducerRoot vespaRoot = new ApplicationConfigProducerRoot(root, "vespa", deployState.getDocumentModel(), @@ -252,6 +257,23 @@ public class StandaloneContainerApplication implements Application { return new Pair<>(root, container); } + private static DeployState createDeployState(ApplicationPackage applicationPackage, FileRegistry fileRegistry, DeployLogger logger) { + DeployState.Builder builder = new DeployState.Builder() + .applicationPackage(applicationPackage) + .fileRegistry(fileRegistry) + .deployLogger(logger) + .configDefinitionRepo(configDefinitionRepo); + + /* Temporarily disable until we know how status.html is updated for config servers/controllers + if (isConfigServer()) + builder.properties(new DeployProperties.Builder() + .hostedVespa(new CloudConfigInstallVariables().hostedVespa().orElse(Boolean.FALSE)) + .build()); + */ + + return builder.build(); + } + private static void initializeContainer(Container container, Element spec) { HostResource host = container.getRoot().getHostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); diff --git a/vagrant/README.md b/vagrant/README.md index d5212692638..8a4ebdc8d60 100644 --- a/vagrant/README.md +++ b/vagrant/README.md @@ -51,7 +51,7 @@ SSH agent forwarding is enabled to ensure easy interaction with GitHub inside th vagrant ssh -#### 5. Checkout vespa source inside virtual machine +#### 6. Checkout vespa source inside virtual machine This is needed in order to compile and run tests fast on the local file system inside the virtual machine. git clone git@github.com:vespa-engine/vespa.git @@ -79,10 +79,12 @@ Open a terminal inside the virtual CentOS desktop (password is "vagrant") and ru clion -When promoted, configure c and cpp compilers to +When prompted, configure toolchains as follows: - /opt/rh/devtoolset-7/root/usr/bin/cc - /opt/rh/devtoolset-7/root/usr/bin/c++ + CMake: /usr/bin/cmake3 + Make: /usr/bin/make + C Compiler: /opt/rh/devtoolset-7/root/usr/bin/cc + C++ Compiler: /opt/rh/devtoolset-7/root/usr/bin/c++ #### 3. Open the Vespa Project Go to *File* -> *Open* and choose <vespa-source>>/CMakeLists.txt. @@ -97,6 +99,9 @@ Under *Build Options* specify "-j 4" and click *Apply*. (Some of the changes made by it are undone by clion on the first startup.) -#### 5. Build all modules +#### 6. Build all modules Choose target **all_modules** from the set of build targets at the top right and click build. +## Starting and stopping the Vagrant machine +Use `vagrant suspend` to suspend the machine and then `vagrant resume` to resume it later on. +Alternatively use `vagrant halt` + `vagrant up` to shutdown and reboot. Latter approach is slower but requires less disk space since RAM content is not persisted to host. diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index f15f45d75a0..996636aeb01 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -61,7 +61,9 @@ Vagrant.configure("2") do |config| firefox \ vim \ emacs - yum-builddep -y /vagrant/dist/vespa.spec + sed -e '/^BuildRequires:/d' -e 's/^Requires:/BuildRequires:/' /vagrant/dist/vespa.spec > /tmp/vesparun.spec + yum-builddep -y /vagrant/dist/vespa.spec /tmp/vesparun.spec + rm /tmp/vesparun.spec echo -e "* soft nproc 409600\n* hard nproc 409600" > /etc/security/limits.d/99-nproc.conf echo -e "* soft nofile 262144\n* hard nofile 262144" > /etc/security/limits.d/99-nofile.conf diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/Extension.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/Extension.java deleted file mode 100644 index 9a6c20018b8..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/Extension.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.tls; - -import org.bouncycastle.asn1.ASN1ObjectIdentifier; - -/** - * @author bjorncs - * @deprecated Use com.yahoo.security.* - */ -@Deprecated -public enum Extension { - BASIC_CONSTRAINS(org.bouncycastle.asn1.x509.Extension.basicConstraints), - SUBJECT_ALTERNATIVE_NAMES(org.bouncycastle.asn1.x509.Extension.subjectAlternativeName); - - final ASN1ObjectIdentifier extensionOId; - - Extension(ASN1ObjectIdentifier extensionOId) { - this.extensionOId = extensionOId; - } - - public String getOId() { - return extensionOId.getId(); - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyAlgorithm.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyAlgorithm.java deleted file mode 100644 index d685f85b206..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyAlgorithm.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.tls; - -/** - * @author bjorncs - * @deprecated Use com.yahoo.security.* - */ -@Deprecated -public enum KeyAlgorithm { - RSA("RSA"); - - private final String algorithmName; - - KeyAlgorithm(String algorithmName) { - this.algorithmName = algorithmName; - } - - String getAlgorithmName() { - return algorithmName; - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilder.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilder.java deleted file mode 100644 index 3e63e441396..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilder.java +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.tls; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.security.GeneralSecurityException; -import java.security.KeyStore; -import java.security.PrivateKey; -import java.security.cert.Certificate; -import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.List; - -import static java.util.Collections.singletonList; - -/** - * @author bjorncs - * @deprecated Use com.yahoo.security.* - */ -@Deprecated -public class KeyStoreBuilder { - - private final List<KeyEntry> keyEntries = new ArrayList<>(); - private final List<CertificateEntry> certificateEntries = new ArrayList<>(); - - private final KeyStoreType keyStoreType; - private File inputFile; - private char[] inputFilePassword; - - private KeyStoreBuilder(KeyStoreType keyStoreType) { - this.keyStoreType = keyStoreType; - } - - public static KeyStoreBuilder withType(KeyStoreType type) { - return new KeyStoreBuilder(type); - } - - public KeyStoreBuilder fromFile(File file, char[] password) { - this.inputFile = file; - this.inputFilePassword = password; - return this; - } - - public KeyStoreBuilder fromFile(File file) { - return fromFile(file, null); - } - - public KeyStoreBuilder withKeyEntry(String alias, PrivateKey privateKey, char[] password, List<X509Certificate> certificateChain) { - keyEntries.add(new KeyEntry(alias, privateKey, certificateChain, password)); - return this; - } - - public KeyStoreBuilder withKeyEntry(String alias, PrivateKey privateKey, char[] password, X509Certificate certificate) { - return withKeyEntry(alias, privateKey, password, singletonList(certificate)); - } - - public KeyStoreBuilder withKeyEntry(String alias, PrivateKey privateKey, X509Certificate certificate) { - return withKeyEntry(alias, privateKey, null, certificate); - } - - public KeyStoreBuilder withKeyEntry(String alias, PrivateKey privateKey, List<X509Certificate> certificateChain) { - return withKeyEntry(alias, privateKey, null, certificateChain); - } - - public KeyStoreBuilder withCertificateEntry(String alias, X509Certificate certificate) { - certificateEntries.add(new CertificateEntry(alias, certificate)); - return this; - } - - public KeyStore build() { - try { - KeyStore keystore = this.keyStoreType.createKeystore(); - if (this.inputFile != null) { - try (InputStream in = new BufferedInputStream(new FileInputStream(this.inputFile))) { - keystore.load(in, this.inputFilePassword); - } - } else { - keystore.load(null); - } - for (KeyEntry entry : keyEntries) { - char[] password = entry.password != null ? entry.password : new char[0]; - Certificate[] certificateChain = entry.certificateChain.toArray(new Certificate[entry.certificateChain.size()]); - keystore.setKeyEntry(entry.alias, entry.privateKey, password, certificateChain); - } - for (CertificateEntry entry : certificateEntries) { - keystore.setCertificateEntry(entry.alias, entry.certificate); - } - return keystore; - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static class KeyEntry { - final String alias; - final PrivateKey privateKey; - final List<X509Certificate> certificateChain; - final char[] password; - - KeyEntry(String alias, PrivateKey privateKey, List<X509Certificate> certificateChain, char[] password) { - this.alias = alias; - this.privateKey = privateKey; - this.certificateChain = certificateChain; - this.password = password; - } - } - - private static class CertificateEntry { - final String alias; - final X509Certificate certificate; - - CertificateEntry(String alias, X509Certificate certificate) { - this.alias = alias; - this.certificate = certificate; - } - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreType.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreType.java deleted file mode 100644 index b0bfe170789..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreType.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.tls; - -import org.bouncycastle.jce.provider.BouncyCastleProvider; - -import java.security.GeneralSecurityException; -import java.security.KeyStore; -import java.security.KeyStoreException; - -/** - * @author bjorncs - * @deprecated Use com.yahoo.security.* - */ -@Deprecated -public enum KeyStoreType { - JKS { - KeyStore createKeystore() throws KeyStoreException { - return KeyStore.getInstance("JKS"); - } - }, - PKCS12 { - KeyStore createKeystore() throws KeyStoreException { - return KeyStore.getInstance("PKCS12", BouncyCastleProviderHolder.getInstance()); - } - }; - abstract KeyStore createKeystore() throws GeneralSecurityException; -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreUtils.java deleted file mode 100644 index 96fe76a1f73..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreUtils.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.tls; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.UncheckedIOException; -import java.security.GeneralSecurityException; -import java.security.KeyStore; - -/** - * @author bjorncs - * @deprecated Use com.yahoo.security.* - */ -@Deprecated -public class KeyStoreUtils { - private KeyStoreUtils() {} - - public static void writeKeyStoreToFile(KeyStore keyStore, File file, char[] password) { - try (OutputStream out = new BufferedOutputStream(new FileOutputStream(file))) { - keyStore.store(out, password); - } catch (IOException e) { - throw new UncheckedIOException(e); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } - - } - - public static void writeKeyStoreToFile(KeyStore keyStore, File file) { - writeKeyStoreToFile(keyStore, file, new char[0]); - } - -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyUtils.java deleted file mode 100644 index fc4734d16ca..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyUtils.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.tls; - -import com.yahoo.athenz.auth.util.Crypto; -import org.bouncycastle.asn1.ASN1Encodable; -import org.bouncycastle.asn1.ASN1Primitive; -import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; -import org.bouncycastle.openssl.PEMKeyPair; -import org.bouncycastle.openssl.PEMParser; -import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; -import org.bouncycastle.openssl.jcajce.JcaPEMWriter; -import org.bouncycastle.util.io.pem.PemObject; - -import java.io.IOException; -import java.io.StringReader; -import java.io.StringWriter; -import java.io.UncheckedIOException; -import java.security.GeneralSecurityException; -import java.security.KeyFactory; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.PrivateKey; -import java.security.PublicKey; -import java.security.spec.PKCS8EncodedKeySpec; - -/** - * @author bjorncs - * @deprecated Use com.yahoo.security.* - */ -@Deprecated -public class KeyUtils { - private KeyUtils() {} - - public static KeyPair generateKeypair(KeyAlgorithm algorithm, int keySize) { - try { - KeyPairGenerator keyGen = KeyPairGenerator.getInstance(algorithm.getAlgorithmName()); - if (keySize != -1) { - keyGen.initialize(keySize); - } - return keyGen.genKeyPair(); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } - } - - public static KeyPair generateKeypair(KeyAlgorithm algorithm) { - return generateKeypair(algorithm, -1); - } - - public static PublicKey extractPublicKey(PrivateKey privateKey) { - return Crypto.extractPublicKey(privateKey); - } - - public static PrivateKey fromPemEncodedPrivateKey(String pem) { - try (PEMParser parser = new PEMParser(new StringReader(pem))) { - Object pemObject = parser.readObject(); - if (pemObject instanceof PrivateKeyInfo) { - PrivateKeyInfo keyInfo = (PrivateKeyInfo) pemObject; - PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyInfo.getEncoded()); - return KeyFactory.getInstance(KeyAlgorithm.RSA.getAlgorithmName()).generatePrivate(keySpec); - } else if (pemObject instanceof PEMKeyPair) { - PEMKeyPair pemKeypair = (PEMKeyPair) pemObject; - PrivateKeyInfo keyInfo = pemKeypair.getPrivateKeyInfo(); - JcaPEMKeyConverter pemConverter = new JcaPEMKeyConverter(); - return pemConverter.getPrivateKey(keyInfo); - } - throw new IllegalArgumentException("Unexpected type of PEM type: " + pemObject); - } catch (IOException e) { - throw new UncheckedIOException(e); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } - } - - public static String toPem(PrivateKey privateKey) { - try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { - // Note: Encoding using PKCS#1 as this is to be read by tools only supporting PKCS#1 - pemWriter.writeObject(new PemObject("RSA PRIVATE KEY", getPkcs1Bytes(privateKey))); - pemWriter.flush(); - return stringWriter.toString(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static byte[] getPkcs1Bytes(PrivateKey privateKey) throws IOException{ - - byte[] privBytes = privateKey.getEncoded(); - PrivateKeyInfo pkInfo = PrivateKeyInfo.getInstance(privBytes); - ASN1Encodable encodable = pkInfo.parsePrivateKey(); - ASN1Primitive primitive = encodable.toASN1Primitive(); - return primitive.getEncoded(); - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/SslContextBuilder.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/SslContextBuilder.java deleted file mode 100644 index 63262eac048..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/SslContextBuilder.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.tls; - -import javax.net.ssl.KeyManager; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.GeneralSecurityException; -import java.security.KeyStore; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; - -/** - * @author bjorncs - * @deprecated Use com.yahoo.security.* - */ -@Deprecated -public class SslContextBuilder { - - private KeyStoreSupplier trustStoreSupplier; - private KeyStoreSupplier keyStoreSupplier; - private char[] keyStorePassword; - - public SslContextBuilder() {} - - public SslContextBuilder withTrustStore(File file, KeyStoreType trustStoreType) { - this.trustStoreSupplier = () -> KeyStoreBuilder.withType(trustStoreType).fromFile(file).build(); - return this; - } - - public SslContextBuilder withTrustStore(KeyStore trustStore) { - this.trustStoreSupplier = () -> trustStore; - return this; - } - - public SslContextBuilder withKeyStore(PrivateKey privateKey, X509Certificate certificate) { - char[] pwd = new char[0]; - this.keyStoreSupplier = () -> KeyStoreBuilder.withType(KeyStoreType.JKS).withKeyEntry("default", privateKey, certificate).build(); - this.keyStorePassword = pwd; - return this; - } - - public SslContextBuilder withKeyStore(KeyStore keyStore, char[] password) { - this.keyStoreSupplier = () -> keyStore; - this.keyStorePassword = password; - return this; - } - - public SslContextBuilder withKeyStore(File file, char[] password, KeyStoreType keyStoreType) { - this.keyStoreSupplier = () -> KeyStoreBuilder.withType(keyStoreType).fromFile(file, password).build(); - this.keyStorePassword = password; - return this; - } - - public SslContextBuilder withKeyStore(File privateKeyPemFile, File certificatePemFile) { - return withKeyStore(privateKeyPemFile.toPath(), certificatePemFile.toPath()); - } - - public SslContextBuilder withKeyStore(Path privateKeyPemFile, Path certificatePemFile) { - this.keyStoreSupplier = - () -> { - PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(new String(Files.readAllBytes(privateKeyPemFile))); - X509Certificate certificate = X509CertificateUtils.fromPem(new String(Files.readAllBytes(certificatePemFile))); - return KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry("default", privateKey, certificate) - .build(); - }; - this.keyStorePassword = new char[0]; - return this; - } - - public SSLContext build() { - try { - SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); - TrustManager[] trustManagers = - trustStoreSupplier != null ? createTrustManagers(trustStoreSupplier) : null; - KeyManager[] keyManagers = - keyStoreSupplier != null ? createKeyManagers(keyStoreSupplier, keyStorePassword) : null; - sslContext.init(keyManagers, trustManagers, null); - return sslContext; - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static TrustManager[] createTrustManagers(KeyStoreSupplier trustStoreSupplier) - throws GeneralSecurityException, IOException { - TrustManagerFactory trustManagerFactory = - TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(trustStoreSupplier.get()); - return trustManagerFactory.getTrustManagers(); - } - - private static KeyManager[] createKeyManagers(KeyStoreSupplier keyStoreSupplier, char[] password) - throws GeneralSecurityException, IOException { - KeyManagerFactory keyManagerFactory = - KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(keyStoreSupplier.get(), password); - return keyManagerFactory.getKeyManagers(); - } - - private interface KeyStoreSupplier { - KeyStore get() throws IOException, GeneralSecurityException; - } - -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/X509CertificateBuilder.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/X509CertificateBuilder.java deleted file mode 100644 index de593f25f61..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/X509CertificateBuilder.java +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.tls; - -import org.bouncycastle.asn1.x509.BasicConstraints; -import org.bouncycastle.asn1.x509.Extension; -import org.bouncycastle.asn1.x509.GeneralName; -import org.bouncycastle.asn1.x509.GeneralNames; -import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; -import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; -import org.bouncycastle.operator.ContentSigner; -import org.bouncycastle.operator.OperatorException; -import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; -import org.bouncycastle.pkcs.PKCS10CertificationRequest; -import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequest; - -import javax.security.auth.x500.X500Principal; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.math.BigInteger; -import java.security.GeneralSecurityException; -import java.security.KeyPair; -import java.security.PrivateKey; -import java.security.PublicKey; -import java.security.cert.X509Certificate; -import java.sql.Date; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; - -import static com.yahoo.vespa.athenz.tls.SubjectAlternativeName.Type.DNS_NAME; - -/** - * @author bjorncs - * @deprecated Use com.yahoo.security.* - */ -@Deprecated -public class X509CertificateBuilder { - - private final long serialNumber; - private final SignatureAlgorithm signingAlgorithm; - private final PrivateKey caPrivateKey; - private final Instant notBefore; - private final Instant notAfter; - private final List<SubjectAlternativeName> subjectAlternativeNames = new ArrayList<>(); - private final X500Principal issuer; - private final X500Principal subject; - private final PublicKey certPublicKey; - private BasicConstraintsExtension basicConstraintsExtension; - - private X509CertificateBuilder(X500Principal issuer, - X500Principal subject, - Instant notBefore, - Instant notAfter, - PublicKey certPublicKey, - PrivateKey caPrivateKey, - SignatureAlgorithm signingAlgorithm, - long serialNumber) { - this.issuer = issuer; - this.subject = subject; - this.notBefore = notBefore; - this.notAfter = notAfter; - this.certPublicKey = certPublicKey; - this.caPrivateKey = caPrivateKey; - this.signingAlgorithm = signingAlgorithm; - this.serialNumber = serialNumber; - } - - public static X509CertificateBuilder fromCsr(Pkcs10Csr csr, - X500Principal caIssuer, - Instant notBefore, - Instant notAfter, - PrivateKey caPrivateKey, - SignatureAlgorithm signingAlgorithm, - long serialNumber) { - try { - PKCS10CertificationRequest bcCsr = csr.getBcCsr(); - PublicKey publicKey = new JcaPKCS10CertificationRequest(bcCsr).getPublicKey(); - return new X509CertificateBuilder(caIssuer, - new X500Principal(bcCsr.getSubject().getEncoded()), - notBefore, - notAfter, - publicKey, - caPrivateKey, - signingAlgorithm, - serialNumber); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static X509CertificateBuilder fromKeypair(KeyPair keyPair, - X500Principal subject, - Instant notBefore, - Instant notAfter, - SignatureAlgorithm signingAlgorithm, - long serialNumber) { - return new X509CertificateBuilder(subject, - subject, - notBefore, - notAfter, - keyPair.getPublic(), - keyPair.getPrivate(), - signingAlgorithm, - serialNumber); - } - - public X509CertificateBuilder addSubjectAlternativeName(String dnsName) { - this.subjectAlternativeNames.add(new SubjectAlternativeName(DNS_NAME, dnsName)); - return this; - } - - public X509CertificateBuilder addSubjectAlternativeName(SubjectAlternativeName san) { - this.subjectAlternativeNames.add(san); - return this; - } - - public X509CertificateBuilder setBasicConstraints(boolean isCritical, boolean isCertAuthorityCertificate) { - this.basicConstraintsExtension = new BasicConstraintsExtension(isCritical, isCertAuthorityCertificate); - return this; - } - - public X509Certificate build() { - try { - JcaX509v3CertificateBuilder jcaCertBuilder = new JcaX509v3CertificateBuilder( - issuer, BigInteger.valueOf(serialNumber), Date.from(notBefore), Date.from(notAfter), subject, certPublicKey); - if (basicConstraintsExtension != null) { - jcaCertBuilder.addExtension( - Extension.basicConstraints, - basicConstraintsExtension.isCritical, - new BasicConstraints(basicConstraintsExtension.isCertAuthorityCertificate)); - } - if (!subjectAlternativeNames.isEmpty()) { - GeneralNames generalNames = new GeneralNames( - subjectAlternativeNames.stream() - .map(SubjectAlternativeName::toGeneralName) - .toArray(GeneralName[]::new)); - jcaCertBuilder.addExtension(Extension.subjectAlternativeName, false, generalNames); - } - ContentSigner contentSigner = new JcaContentSignerBuilder(signingAlgorithm.getAlgorithmName()) - .setProvider(BouncyCastleProviderHolder.getInstance()) - .build(caPrivateKey); - return new JcaX509CertificateConverter() - .setProvider(BouncyCastleProviderHolder.getInstance()) - .getCertificate(jcaCertBuilder.build(contentSigner)); - } catch (OperatorException | GeneralSecurityException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/X509CertificateUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/X509CertificateUtils.java deleted file mode 100644 index 8fc25ab06a4..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/X509CertificateUtils.java +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.tls; - -import org.bouncycastle.asn1.ASN1Encodable; -import org.bouncycastle.asn1.ASN1OctetString; -import org.bouncycastle.asn1.ASN1Primitive; -import org.bouncycastle.asn1.x509.GeneralNames; -import org.bouncycastle.cert.X509CertificateHolder; -import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; -import org.bouncycastle.openssl.PEMParser; -import org.bouncycastle.openssl.jcajce.JcaPEMWriter; -import org.bouncycastle.util.io.pem.PemObject; - -import javax.naming.NamingException; -import javax.naming.ldap.LdapName; -import javax.security.auth.x500.X500Principal; -import java.io.IOException; -import java.io.StringReader; -import java.io.StringWriter; -import java.io.UncheckedIOException; -import java.security.GeneralSecurityException; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static com.yahoo.vespa.athenz.tls.Extension.SUBJECT_ALTERNATIVE_NAMES; -import static java.util.stream.Collectors.toList; - -/** - * @author bjorncs - * @deprecated Use com.yahoo.security.* - */ -@Deprecated -public class X509CertificateUtils { - - private X509CertificateUtils() {} - - public static X509Certificate fromPem(String pem) { - try (PEMParser parser = new PEMParser(new StringReader(pem))) { - return toX509Certificate(parser.readObject()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } catch (CertificateException e) { - throw new RuntimeException(e); - } - } - - public static List<X509Certificate> certificateListFromPem(String pem) { - try (PEMParser parser = new PEMParser(new StringReader(pem))) { - List<X509Certificate> list = new ArrayList<>(); - Object pemObject; - while ((pemObject = parser.readObject()) != null) { - list.add(toX509Certificate(pemObject)); - } - return list; - } catch (IOException e) { - throw new UncheckedIOException(e); - } catch (CertificateException e) { - throw new RuntimeException(e); - } - } - - private static X509Certificate toX509Certificate(Object pemObject) throws CertificateException { - if (pemObject instanceof X509Certificate) { - return (X509Certificate) pemObject; - } - if (pemObject instanceof X509CertificateHolder) { - return new JcaX509CertificateConverter() - .setProvider(BouncyCastleProviderHolder.getInstance()) - .getCertificate((X509CertificateHolder) pemObject); - } - throw new IllegalArgumentException("Invalid type of PEM object: " + pemObject); - } - - public static String toPem(X509Certificate certificate) { - try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { - pemWriter.writeObject(new PemObject("CERTIFICATE", certificate.getEncoded())); - pemWriter.flush(); - return stringWriter.toString(); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static String toPem(List<X509Certificate> certificates) { - try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { - for (X509Certificate certificate : certificates) { - pemWriter.writeObject(new PemObject("CERTIFICATE", certificate.getEncoded())); - } - pemWriter.flush(); - return stringWriter.toString(); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static List<String> getSubjectCommonNames(X509Certificate certificate) { - return getCommonNames(certificate.getSubjectX500Principal()); - } - - public static List<String> getIssuerCommonNames(X509Certificate certificate) { - return getCommonNames(certificate.getIssuerX500Principal()); - } - - public static List<String> getCommonNames(X500Principal subject) { - try { - String subjectPrincipal = subject.getName(); - return new LdapName(subjectPrincipal).getRdns().stream() - .filter(rdn -> rdn.getType().equalsIgnoreCase("cn")) - .map(rdn -> rdn.getValue().toString()) - .collect(toList()); - } catch (NamingException e) { - throw new IllegalArgumentException("Invalid CN: " + e, e); - } - - } - - public static List<SubjectAlternativeName> getSubjectAlternativeNames(X509Certificate certificate) { - try { - byte[] extensionValue = certificate.getExtensionValue(SUBJECT_ALTERNATIVE_NAMES.getOId()); - if (extensionValue == null) return Collections.emptyList(); - ASN1Encodable asn1Encodable = ASN1Primitive.fromByteArray(extensionValue); - if (asn1Encodable instanceof ASN1OctetString) { - asn1Encodable = ASN1Primitive.fromByteArray(((ASN1OctetString) asn1Encodable).getOctets()); - } - GeneralNames names = GeneralNames.getInstance(asn1Encodable); - return SubjectAlternativeName.fromGeneralNames(names); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilderTest.java deleted file mode 100644 index 6060f6f3521..00000000000 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilderTest.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.yahoo.vespa.athenz.tls; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.security.KeyPair; -import java.security.cert.X509Certificate; - -import static com.yahoo.vespa.athenz.tls.TestUtils.createCertificate; -import static com.yahoo.vespa.athenz.tls.TestUtils.createKeystoreFile; - -/** - * @author bjorncs - */ -public class KeyStoreBuilderTest { - - private static final char[] PASSWORD = new char[0]; - - @Rule - public TemporaryFolder tempDirectory = new TemporaryFolder(); - - @Test - public void can_create_jks_keystore_from_privatekey_and_certificate() throws Exception { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 4096); - X509Certificate certificate = createCertificate(keyPair); - KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry("key", keyPair.getPrivate(), certificate) - .build(); - } - - @Test - public void can_build_jks_keystore_from_file() throws Exception { - File keystoreFile = tempDirectory.newFile(); - createKeystoreFile(keystoreFile, KeyStoreType.JKS, PASSWORD); - - KeyStoreBuilder.withType(KeyStoreType.JKS) - .fromFile(keystoreFile, PASSWORD) - .build(); - } - - @Test - public void can_build_pcks12_keystore_from_file() throws Exception { - File keystoreFile = tempDirectory.newFile(); - createKeystoreFile(keystoreFile, KeyStoreType.PKCS12, PASSWORD); - - KeyStoreBuilder.withType(KeyStoreType.PKCS12) - .fromFile(keystoreFile, PASSWORD) - .build(); - } - -}
\ No newline at end of file diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyUtilsTest.java deleted file mode 100644 index fbdc6f1e3bd..00000000000 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyUtilsTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.yahoo.vespa.athenz.tls; - -import org.junit.Test; - -import java.security.KeyPair; -import java.security.PrivateKey; -import java.security.PublicKey; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; - -/** - * @author bjorncs - */ -public class KeyUtilsTest { - - @Test - public void can_extract_public_key_from_private() { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - PublicKey publicKey = KeyUtils.extractPublicKey(keyPair.getPrivate()); - assertNotNull(publicKey); - } - - @Test - public void can_serialize_deserialize_pem() { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - String pem = KeyUtils.toPem(keyPair.getPrivate()); - assertThat(pem, containsString("BEGIN RSA PRIVATE KEY")); - assertThat(pem, containsString("END RSA PRIVATE KEY")); - PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem); - assertEquals(keyPair.getPrivate(), deserializedKey); - } - -}
\ No newline at end of file diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrBuilderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrBuilderTest.java index e3aaba66efe..3a00ad6a7a4 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrBuilderTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrBuilderTest.java @@ -1,5 +1,7 @@ package com.yahoo.vespa.athenz.tls; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; import org.junit.Test; import javax.security.auth.x500.X500Principal; diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrTest.java index ea60511f39c..8213856512d 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrTest.java @@ -1,5 +1,8 @@ package com.yahoo.vespa.athenz.tls; +import com.yahoo.security.Extension; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; import org.junit.Test; import javax.security.auth.x500.X500Principal; @@ -48,7 +51,7 @@ public class Pkcs10CsrTest { .addSubjectAlternativeName("san") .setBasicConstraints(true, true) .build(); - List<String> expected = Arrays.asList(Extension.BASIC_CONSTRAINS.getOId(), Extension.SUBJECT_ALTERNATIVE_NAMES.getOId()); + List<String> expected = Arrays.asList(Extension.BASIC_CONSTRAINTS.getOId(), Extension.SUBJECT_ALTERNATIVE_NAMES.getOId()); List<String> actual = csr.getExtensionOIds(); assertEquals(expected, actual); } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrUtilsTest.java index 5b5a57f1fcc..fcbc6d00a8e 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrUtilsTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/Pkcs10CsrUtilsTest.java @@ -1,5 +1,7 @@ package com.yahoo.vespa.athenz.tls; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; import org.junit.Test; import javax.security.auth.x500.X500Principal; diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/SslContextBuilderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/SslContextBuilderTest.java deleted file mode 100644 index 2f750d915d4..00000000000 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/SslContextBuilderTest.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.tls; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.security.KeyPair; -import java.security.cert.X509Certificate; - -import static com.yahoo.vespa.athenz.tls.TestUtils.createCertificate; -import static com.yahoo.vespa.athenz.tls.TestUtils.createKeystore; -import static com.yahoo.vespa.athenz.tls.TestUtils.createKeystoreFile; - -/** - * @author bjorncs - */ -public class SslContextBuilderTest { - - private static final char[] PASSWORD = new char[0]; - - @Rule - public TemporaryFolder tempDirectory = new TemporaryFolder(); - - @Test - public void can_build_sslcontext_with_truststore_only() throws Exception { - new SslContextBuilder() - .withTrustStore(createKeystore(KeyStoreType.JKS, PASSWORD)) - .build(); - } - - @Test - public void can_build_sslcontext_with_keystore_only() throws Exception { - new SslContextBuilder() - .withKeyStore(createKeystore(KeyStoreType.JKS, PASSWORD), PASSWORD) - .build(); - } - - @Test - public void can_build_sslcontext_with_truststore_and_keystore() throws Exception { - new SslContextBuilder() - .withKeyStore(createKeystore(KeyStoreType.JKS, PASSWORD), PASSWORD) - .withTrustStore(createKeystore(KeyStoreType.JKS, PASSWORD)) - .build(); - } - - @Test - public void can_build_sslcontext_with_keystore_from_private_key_and_certificate() throws Exception { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - X509Certificate certificate = createCertificate(keyPair); - new SslContextBuilder() - .withKeyStore(keyPair.getPrivate(), certificate) - .build(); - } - - @Test - public void can_build_sslcontext_with_jks_keystore_from_file() throws Exception { - File keystoreFile = tempDirectory.newFile(); - createKeystoreFile(keystoreFile, KeyStoreType.JKS, PASSWORD); - - new SslContextBuilder() - .withKeyStore(keystoreFile, PASSWORD, KeyStoreType.JKS) - .build(); - } - - @Test - public void can_build_sslcontext_with_pcks12_keystore_from_file() throws Exception { - File keystoreFile = tempDirectory.newFile(); - createKeystoreFile(keystoreFile, KeyStoreType.PKCS12, PASSWORD); - - new SslContextBuilder() - .withKeyStore(keystoreFile, PASSWORD, KeyStoreType.PKCS12) - .build(); - } - -} diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/TestUtils.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/TestUtils.java index 2a9b54f9e9e..048538c1a33 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/TestUtils.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/TestUtils.java @@ -1,15 +1,21 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.tls; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyStoreBuilder; +import com.yahoo.security.KeyStoreType; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateBuilder; + import javax.security.auth.x500.X500Principal; -import java.io.File; +import java.math.BigInteger; import java.security.KeyPair; import java.security.KeyStore; import java.security.cert.X509Certificate; import java.time.Instant; import java.time.temporal.ChronoUnit; -import static com.yahoo.vespa.athenz.tls.KeyStoreUtils.writeKeyStoreToFile; +import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_RSA; /** * @author bjorncs @@ -30,11 +36,8 @@ class TestUtils { static X509Certificate createCertificate(KeyPair keyPair, X500Principal subject) { return X509CertificateBuilder .fromKeypair( - keyPair, subject, Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA256_WITH_RSA, 1) + keyPair, subject, Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SHA256_WITH_RSA, BigInteger.ONE) .build(); } - static void createKeystoreFile(File file, KeyStoreType type, char[] password) { - writeKeyStoreToFile(createKeystore(type, password), file, password); - } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/X509CertificateBuilderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/X509CertificateBuilderTest.java deleted file mode 100644 index 81ff4fdb208..00000000000 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/X509CertificateBuilderTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.yahoo.vespa.athenz.tls; - -import org.junit.Test; - -import javax.security.auth.x500.X500Principal; -import java.security.KeyPair; -import java.security.NoSuchAlgorithmException; -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -import static org.junit.Assert.assertEquals; - -/** - * @author bjorncs - */ -public class X509CertificateBuilderTest { - - @Test - public void can_build_self_signed_certificate() throws NoSuchAlgorithmException { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - X500Principal subject = new X500Principal("CN=myservice"); - X509Certificate cert = - X509CertificateBuilder.fromKeypair( - keyPair, - subject, - Instant.now(), - Instant.now().plus(1, ChronoUnit.DAYS), - SignatureAlgorithm.SHA256_WITH_RSA, - 1) - .setBasicConstraints(true, true) - .build(); - assertEquals(subject, cert.getSubjectX500Principal()); - } - - @Test - public void can_build_certificate_from_csr() { - X500Principal subject = new X500Principal("CN=subject"); - X500Principal issuer = new X500Principal("CN=issuer"); - KeyPair csrKeypair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(subject, csrKeypair, SignatureAlgorithm.SHA256_WITH_RSA).build(); - KeyPair caKeypair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - X509Certificate cert = X509CertificateBuilder - .fromCsr( - csr, - issuer, - Instant.now(), - Instant.now().plus(1, ChronoUnit.DAYS), - caKeypair.getPrivate(), - SignatureAlgorithm.SHA256_WITH_RSA, - 1) - .addSubjectAlternativeName("subject1.alt") - .addSubjectAlternativeName("subject2.alt") - .build(); - assertEquals(subject, cert.getSubjectX500Principal()); - } - -}
\ No newline at end of file diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/X509CertificateUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/X509CertificateUtilsTest.java deleted file mode 100644 index 4039bf36a5f..00000000000 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/X509CertificateUtilsTest.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.yahoo.vespa.athenz.tls; - -import org.junit.Test; - -import javax.security.auth.x500.X500Principal; -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Arrays; -import java.util.List; - -import static com.yahoo.vespa.athenz.tls.SubjectAlternativeName.Type.DNS_NAME; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -/** - * @author bjorncs - */ -public class X509CertificateUtilsTest { - @Test - public void can_deserialize_serialized_pem_certificate() { - KeyPair keypair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - X500Principal subject = new X500Principal("CN=myservice"); - X509Certificate cert = TestUtils.createCertificate(keypair, subject); - assertEquals(subject, cert.getSubjectX500Principal()); - String pem = X509CertificateUtils.toPem(cert); - assertThat(pem, containsString("BEGIN CERTIFICATE")); - assertThat(pem, containsString("END CERTIFICATE")); - X509Certificate deserializedCert = X509CertificateUtils.fromPem(pem); - assertEquals(subject, deserializedCert.getSubjectX500Principal()); - } - - @Test - public void can_deserialize_serialized_pem_certificate_list() { - KeyPair keypair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - X500Principal subject1 = new X500Principal("CN=myservice"); - X509Certificate cert1 = TestUtils.createCertificate(keypair, subject1); - X500Principal subject2 = new X500Principal("CN=myservice"); - X509Certificate cert2 = TestUtils.createCertificate(keypair, subject2); - List<X509Certificate> certificateList = Arrays.asList(cert1, cert2); - String pem = X509CertificateUtils.toPem(certificateList); - List<X509Certificate> deserializedCertificateList = X509CertificateUtils.certificateListFromPem(pem); - assertEquals(2, certificateList.size()); - assertEquals(subject1, deserializedCertificateList.get(0).getSubjectX500Principal()); - assertEquals(subject2, deserializedCertificateList.get(1).getSubjectX500Principal()); - } - - @Test - public void can_list_subject_alternative_names() { - KeyPair keypair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - X500Principal subject = new X500Principal("CN=myservice"); - SubjectAlternativeName san = new SubjectAlternativeName(DNS_NAME, "dns-san"); - X509Certificate cert = X509CertificateBuilder - .fromKeypair( - keypair, - subject, - Instant.now(), - Instant.now().plus(1, ChronoUnit.DAYS), - SignatureAlgorithm.SHA256_WITH_RSA, - 1) - .addSubjectAlternativeName(san) - .build(); - - List<SubjectAlternativeName> sans = X509CertificateUtils.getSubjectAlternativeNames(cert); - assertThat(sans.size(), is(1)); - assertThat(sans.get(0), equalTo(san)); - } -}
\ No newline at end of file diff --git a/vespa-documentgen-plugin/etc/complex/book.sd b/vespa-documentgen-plugin/etc/complex/book.sd index 2635ebe9881..16bf4447979 100644 --- a/vespa-documentgen-plugin/etc/complex/book.sd +++ b/vespa-documentgen-plugin/etc/complex/book.sd @@ -30,7 +30,7 @@ search book { attribute: prefetch } - field mynestedwsfloat type weightedset<weightedset<float>> {} + field mynestedwsfloat type weightedset<float> {} field myarrayint type array<int> { indexing: attribute diff --git a/vespa-documentgen-plugin/etc/complex/common.sd b/vespa-documentgen-plugin/etc/complex/common.sd index 0764421ac8d..e0505eba05b 100644 --- a/vespa-documentgen-plugin/etc/complex/common.sd +++ b/vespa-documentgen-plugin/etc/complex/common.sd @@ -17,19 +17,13 @@ search common { indexing: summary } field weight type float { - indexing { - input weight * 10 | attribute | summary; - } + indexing: attribute | summary } field w1 type float { - indexing { - input weight * 6 + input w1 | summary; - } + indexing: summary } field w2 type float { - indexing { - input w2 + input weight | summary; - } + indexing: summary } field did type string { indexing: attribute|index|summary diff --git a/vespa-documentgen-plugin/etc/complex/common2.sd b/vespa-documentgen-plugin/etc/complex/common2.sd new file mode 100644 index 00000000000..e32d3ed6751 --- /dev/null +++ b/vespa-documentgen-plugin/etc/complex/common2.sd @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search common2 { + document { + field com2 type string { + + } + } +} + diff --git a/vespa-documentgen-plugin/etc/complex/music2.sd b/vespa-documentgen-plugin/etc/complex/music2.sd index 5657580e622..2e2d96ecdec 100644 --- a/vespa-documentgen-plugin/etc/complex/music2.sd +++ b/vespa-documentgen-plugin/etc/complex/music2.sd @@ -56,7 +56,7 @@ search music2 { } field didinteger type array<int> { - indexing: input did | split " " | attribute + indexing: input did | split " " | for_each { to_int } | attribute } rank-profile default { diff --git a/vespa-documentgen-plugin/etc/complex/music3.sd b/vespa-documentgen-plugin/etc/complex/music3.sd new file mode 100644 index 00000000000..65f37029d04 --- /dev/null +++ b/vespa-documentgen-plugin/etc/complex/music3.sd @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search music3 { + document music3 inherits music2, common2 { + field mu3 type string { + + } + } +} diff --git a/vespa-documentgen-plugin/etc/complex/video.sd b/vespa-documentgen-plugin/etc/complex/video.sd index dbaba54cc45..0b0298a162c 100644 --- a/vespa-documentgen-plugin/etc/complex/video.sd +++ b/vespa-documentgen-plugin/etc/complex/video.sd @@ -37,7 +37,7 @@ search video { rank-profile default { first-phase { - expression: nativeRank + expression: file:non-existing.expression } } rank-profile rp1 inherits default { diff --git a/vespa-documentgen-plugin/etc/localapp/common.sd b/vespa-documentgen-plugin/etc/localapp/common.sd index ada7ce7436a..724897b4e7f 100644 --- a/vespa-documentgen-plugin/etc/localapp/common.sd +++ b/vespa-documentgen-plugin/etc/localapp/common.sd @@ -17,19 +17,13 @@ search common { indexing: summary } field weight type float { - indexing { - input weight * 10 | attribute | summary; - } + indexing: attribute|summary } field w1 type float { - indexing { - input weight * 6 + input w1 | summary; - } + indexing: summary } field w2 type float { - indexing { - input w2 + input weight | summary; - } + indexing: summary } field did type string { indexing: attribute|index|summary diff --git a/vespa-documentgen-plugin/etc/localapp/music.sd b/vespa-documentgen-plugin/etc/localapp/music.sd index 0cfe5cf923a..e00e046f511 100644 --- a/vespa-documentgen-plugin/etc/localapp/music.sd +++ b/vespa-documentgen-plugin/etc/localapp/music.sd @@ -51,7 +51,7 @@ search music { } field didinteger type array<int> { - indexing: input did | split " " | attribute + indexing: input did | split " " | for_each { to_int } | attribute } rank-profile default { diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index eab3983dc69..d27352b8ea7 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa; +import com.yahoo.collections.Pair; import com.yahoo.document.*; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; @@ -8,9 +9,7 @@ import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.UnprocessingSearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.tensor.TensorType; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -78,10 +77,10 @@ public class DocumentGenMojo extends AbstractMojo { void execute(File sdDir, File outputDir, String packageName) throws MojoFailureException { if ("".equals(packageName)) throw new IllegalArgumentException("You may not use empty package for generated types."); - searches = new HashMap<String, Search>(); - docTypes = new HashMap<String, String>(); - structTypes = new HashMap<String, String>(); - annotationTypes = new HashMap<String, String>(); + searches = new HashMap<>(); + docTypes = new HashMap<>(); + structTypes = new HashMap<>(); + annotationTypes = new HashMap<>(); outputDir.mkdirs(); SearchBuilder builder = buildSearches(sdDir); @@ -109,7 +108,7 @@ public class DocumentGenMojo extends AbstractMojo { public boolean accept(File dir, String name) { return name.endsWith(".sd"); }}); - SearchBuilder builder = new UnprocessingSearchBuilder(); + SearchBuilder builder = new SearchBuilder(true); for (File f : sdFiles) { try { long modTime = f.lastModified(); @@ -405,7 +404,9 @@ public class DocumentGenMojo extends AbstractMojo { */ private void exportDocumentClass(NewDocumentType docType, Writer out, String packageName) throws IOException { String className = className(docType.getName()); - String superType = javaSuperType(docType); + Pair<String, Boolean> extendInfo = javaSuperType(docType); + String superType = extendInfo.getFirst(); + Boolean multiExtends = extendInfo.getSecond(); out.write( "package "+packageName+";\n\n" + exportInnerImportsFromSuperTypes(docType, packageName) + @@ -441,12 +442,14 @@ public class DocumentGenMojo extends AbstractMojo { exportStructTypeGetter(docType.getName()+".header", docType.allHeader().getFields(), out, 1, "getHeaderStructType", "com.yahoo.document.StructDataType"); exportStructTypeGetter(docType.getName()+".body", docType.allBody().getFields(), out, 1, "getBodyStructType", "com.yahoo.document.StructDataType"); - exportExtendedStructTypeGetter(className, docType.getName(), docType.getAllFields(), out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); - exportCopyConstructor(className, docType.getAllFields(), out, 1, true); - exportFieldsAndAccessors(className, "com.yahoo.document.Document".equals(superType) ? docType.getAllFields() : docType.getFields(), out, 1, true); - exportDocumentMethods(docType.getAllFields(), out, 1); - exportHashCode(docType.getAllFields(), out, 1, "(getDataType() != null ? getDataType().hashCode() : 0) + getId().hashCode()"); - exportEquals(className, docType.getAllFields(), out, 1); + Collection<Field> allUniqueFields = getAllUniqueFields(multiExtends, docType.getAllFields()); + exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); + exportCopyConstructor(className, allUniqueFields, out, 1, true); + + exportFieldsAndAccessors(className, "com.yahoo.document.Document".equals(superType) ? allUniqueFields : docType.getFields(), out, 1, true); + exportDocumentMethods(allUniqueFields, out, 1); + exportHashCode(allUniqueFields, out, 1, "(getDataType() != null ? getDataType().hashCode() : 0) + getId().hashCode()"); + exportEquals(className, allUniqueFields, out, 1); Set<DataType> exportedStructs = exportStructTypes(docType.getTypes(), out, 1, null); docTypes.put(docType.getName(), packageName+"."+className); for (DataType exportedStruct : exportedStructs) { @@ -455,15 +458,36 @@ public class DocumentGenMojo extends AbstractMojo { out.write("}\n"); } + private Collection<Field> getAllUniqueFields(Boolean multipleInheritance, Collection<Field> allFields) { + if (multipleInheritance) { + Map<String, Field> seen = new HashMap<>(); + List<Field> unique = new ArrayList<>(allFields.size()); + for (Field f : allFields) { + if (seen.containsKey(f.getName())) { + if ( ! f.equals(seen.get(f.getName()))) { + throw new IllegalArgumentException("Field '" + f.getName() + "' has conflicting definitions in multiple inheritance." + + "First defined as '" + seen.get(f.getName()) + "', then as '" + f + "'."); + } + } else { + unique.add(f); + seen.put(f.getName(), f); + } + } + return unique; + } + return allFields; + } + /** * The Java class the class of the given type should inherit from. If the input type inherits from _one_ * other type, use that, otherwise Document. */ - private static String javaSuperType(NewDocumentType docType) { + private static Pair<String,Boolean> javaSuperType(NewDocumentType docType) { String ret = "com.yahoo.document.Document"; Collection<NewDocumentType> specInheriteds = specificInheriteds(docType); - if (!specInheriteds.isEmpty() && singleInheritance(specInheriteds)) ret = className(specInheriteds.iterator().next().getName()); - return ret; + boolean singleExtends = singleInheritance(specInheriteds); + if (!specInheriteds.isEmpty() && singleExtends) ret = className(specInheriteds.iterator().next().getName()); + return new Pair<>(ret, !singleExtends); } private static boolean singleInheritance(Collection<NewDocumentType> specInheriteds) { diff --git a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java index a9a5893cf96..b21f38c586a 100644 --- a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java +++ b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java @@ -33,6 +33,7 @@ public class DocumentGenTest { mojo.execute(new File("etc/complex/"), new File("target/generated-test-sources/vespa-documentgen-plugin/"), "com.yahoo.vespa.document"); Map<String, Search> searches = mojo.getSearches(); assertEquals(searches.get("video").getDocument("video").getField("weight").getDataType(), DataType.FLOAT); + assertEquals(searches.get("book").getDocument("book").getField("sw1").getDataType(), DataType.FLOAT); assertTrue(searches.get("book").getDocument("book").getField("mystruct").getDataType() instanceof StructDataType); assertTrue(searches.get("book").getDocument("book").getField("mywsfloat").getDataType() instanceof WeightedSetDataType); assertTrue(((WeightedSetDataType)(searches.get("book").getDocument("book").getField("mywsfloat").getDataType())).getNestedType() == DataType.FLOAT); diff --git a/vespajlib/src/main/java/com/yahoo/security/KeyUtils.java b/vespajlib/src/main/java/com/yahoo/security/KeyUtils.java index 1c3157d639f..11fb0f432e4 100644 --- a/vespajlib/src/main/java/com/yahoo/security/KeyUtils.java +++ b/vespajlib/src/main/java/com/yahoo/security/KeyUtils.java @@ -35,7 +35,6 @@ import static com.yahoo.security.KeyAlgorithm.RSA; /** * @author bjorncs */ -// TODO Support serialization of EC private keys public class KeyUtils { private KeyUtils() {} @@ -88,7 +87,7 @@ public class KeyUtils { } else if (pemObject instanceof PEMKeyPair) { PEMKeyPair pemKeypair = (PEMKeyPair) pemObject; PrivateKeyInfo keyInfo = pemKeypair.getPrivateKeyInfo(); - JcaPEMKeyConverter pemConverter = new JcaPEMKeyConverter(); + JcaPEMKeyConverter pemConverter = new JcaPEMKeyConverter().setProvider(BouncyCastleProviderHolder.getInstance()); return pemConverter.getPrivateKey(keyInfo); } throw new IllegalArgumentException("Unexpected type of PEM type: " + pemObject); @@ -101,8 +100,17 @@ public class KeyUtils { public static String toPem(PrivateKey privateKey) { try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { + String algorithm = privateKey.getAlgorithm(); // Note: Encoding using PKCS#1 as this is to be read by tools only supporting PKCS#1 - pemWriter.writeObject(new PemObject("RSA PRIVATE KEY", getPkcs1Bytes(privateKey))); + String type; + if (algorithm.equals(RSA.getAlgorithmName())) { + type = "RSA PRIVATE KEY"; + } else if (algorithm.equals(EC.getAlgorithmName())) { + type = "EC PRIVATE KEY"; + } else { + throw new IllegalArgumentException("Unexpected key algorithm: " + algorithm); + } + pemWriter.writeObject(new PemObject(type, getPkcs1Bytes(privateKey))); pemWriter.flush(); return stringWriter.toString(); } catch (IOException e) { diff --git a/vespajlib/src/test/java/com/yahoo/security/KeyUtilsTest.java b/vespajlib/src/test/java/com/yahoo/security/KeyUtilsTest.java index 825f4446d94..5e786654d7c 100644 --- a/vespajlib/src/test/java/com/yahoo/security/KeyUtilsTest.java +++ b/vespajlib/src/test/java/com/yahoo/security/KeyUtilsTest.java @@ -32,7 +32,7 @@ public class KeyUtilsTest { } @Test - public void can_serialize_deserialize_pem() { + public void can_serialize_and_deserialize_rsa_privatekey_using_pem_format() { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); String pem = KeyUtils.toPem(keyPair.getPrivate()); assertThat(pem, containsString("BEGIN RSA PRIVATE KEY")); @@ -41,4 +41,14 @@ public class KeyUtilsTest { assertEquals(keyPair.getPrivate(), deserializedKey); } + @Test + public void can_serialize_and_deserialize_ec_privatekey_using_pem_format() { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC); + String pem = KeyUtils.toPem(keyPair.getPrivate()); + assertThat(pem, containsString("BEGIN EC PRIVATE KEY")); + assertThat(pem, containsString("END EC PRIVATE KEY")); + PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem); + assertEquals(keyPair.getPrivate(), deserializedKey); + } + }
\ No newline at end of file diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 4ae98be29b6..fb3b08b325f 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -33,6 +33,7 @@ vespa_define_module( src/tests/data/memory_input src/tests/data/output_writer src/tests/data/simple_buffer + src/tests/data/smart_buffer src/tests/delegatelist src/tests/dotproduct src/tests/dual_merge_director diff --git a/vespalib/src/tests/data/smart_buffer/CMakeLists.txt b/vespalib/src/tests/data/smart_buffer/CMakeLists.txt new file mode 100644 index 00000000000..e7468f4f508 --- /dev/null +++ b/vespalib/src/tests/data/smart_buffer/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_smart_buffer_test_app TEST + SOURCES + smart_buffer_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_smart_buffer_test_app COMMAND vespalib_smart_buffer_test_app) diff --git a/vespalib/src/tests/data/smart_buffer/smart_buffer_test.cpp b/vespalib/src/tests/data/smart_buffer/smart_buffer_test.cpp new file mode 100644 index 00000000000..360afba091a --- /dev/null +++ b/vespalib/src/tests/data/smart_buffer/smart_buffer_test.cpp @@ -0,0 +1,133 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/data/smart_buffer.h> + +using namespace vespalib; + +void checkMemory(const vespalib::string &expect, const Memory &mem) { + EXPECT_EQUAL(expect, vespalib::string(mem.data, mem.size)); +} + +void checkBuffer(const vespalib::string &expect, SmartBuffer &buf) { + TEST_DO(checkMemory(expect, buf.obtain())); +} + +void write_buf(const vespalib::string &str, SmartBuffer &buf) { + WritableMemory mem = buf.reserve(str.size()); + for (size_t i = 0; i < str.size(); ++i) { + mem.data[i] = str.data()[i]; + } + buf.commit(str.size()); +} + +TEST("require that basic read/write works") { + SmartBuffer buf(3); + TEST_DO(checkBuffer("", buf)); + { // read from empty buffer + EXPECT_EQUAL(0u, buf.obtain().size); + } + { // write to buffer + WritableMemory mem = buf.reserve(10); + TEST_DO(checkBuffer("", buf)); + EXPECT_LESS_EQUAL(10u, mem.size); + mem.data[0] = 'a'; + mem.data[1] = 'b'; + mem.data[2] = 'c'; + EXPECT_EQUAL(&buf, &buf.commit(3)); + mem = buf.reserve(0); + TEST_DO(checkBuffer("abc", buf)); + EXPECT_LESS_EQUAL(0u, mem.size); + } + { // read without evicting last byte + Memory mem = buf.obtain(); + TEST_DO(checkBuffer("abc", buf)); + TEST_DO(checkMemory("abc", mem)); + EXPECT_EQUAL(&buf, &buf.evict(2)); + mem = buf.obtain(); + TEST_DO(checkBuffer("c", buf)); + TEST_DO(checkMemory("c", mem)); + mem = buf.obtain(); + TEST_DO(checkBuffer("c", buf)); + TEST_DO(checkMemory("c", mem)); + } + { // write more to buffer + WritableMemory mem = buf.reserve(10); + EXPECT_LESS_EQUAL(10u, mem.size); + TEST_DO(checkBuffer("c", buf)); + mem.data[0] = 'd'; + EXPECT_EQUAL(&buf, &buf.commit(1)); + mem = buf.reserve(5); + TEST_DO(checkBuffer("cd", buf)); + EXPECT_LESS_EQUAL(5u, mem.size); + } + { // read until end + Memory mem = buf.obtain(); + TEST_DO(checkBuffer("cd", buf)); + TEST_DO(checkMemory("cd", mem)); + EXPECT_EQUAL(&buf, &buf.evict(1)); + mem = buf.obtain(); + TEST_DO(checkBuffer("d", buf)); + TEST_DO(checkMemory("d", mem)); + EXPECT_EQUAL(&buf, &buf.evict(1)); + mem = buf.obtain(); + TEST_DO(checkBuffer("", buf)); + TEST_DO(checkMemory("", mem)); + } +} + +TEST("require that requested initial size is not adjusted") { + SmartBuffer buf(400); + EXPECT_EQUAL(buf.capacity(), 400u); +} + +TEST("require that buffer auto-resets when empty") { + SmartBuffer buf(64); + EXPECT_EQUAL(buf.reserve(10).size, 64u); + write_buf("abc", buf); + EXPECT_EQUAL(buf.reserve(10).size, 61u); + buf.evict(3); + EXPECT_EQUAL(buf.reserve(10).size, 64u); +} + +TEST("require that buffer can grow") { + SmartBuffer buf(64); + EXPECT_EQUAL(buf.capacity(), 64u); + write_buf("abc", buf); + write_buf("abc", buf); + buf.evict(3); + EXPECT_EQUAL(buf.reserve(70).size, size_t(128 - 3)); + TEST_DO(checkBuffer("abc", buf)); + EXPECT_EQUAL(buf.capacity(), 128u); +} + +TEST("require that buffer can grow more than 2x") { + SmartBuffer buf(64); + EXPECT_EQUAL(buf.capacity(), 64u); + write_buf("abc", buf); + write_buf("abc", buf); + buf.evict(3); + EXPECT_EQUAL(buf.reserve(170).size, 170u); + TEST_DO(checkBuffer("abc", buf)); + EXPECT_EQUAL(buf.capacity(), 173u); +} + +TEST("require that buffer can be compacted") { + SmartBuffer buf(16); + EXPECT_EQUAL(buf.capacity(), 16u); + write_buf("abc", buf); + write_buf("abc", buf); + buf.evict(3); + write_buf("abc", buf); + buf.evict(3); + write_buf("abc", buf); + buf.evict(3); + write_buf("abc", buf); + buf.evict(3); + EXPECT_EQUAL(buf.reserve(0).size, 1u); + write_buf("abc", buf); + TEST_DO(checkBuffer("abcabc", buf)); + EXPECT_EQUAL(buf.capacity(), 16u); + EXPECT_EQUAL(buf.reserve(0).size, 10u); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/data/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/CMakeLists.txt index 3a94e00ae33..517d0cd198f 100644 --- a/vespalib/src/vespa/vespalib/data/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/data/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_add_library(vespalib_vespalib_data OBJECT output.cpp output_writer.cpp simple_buffer.cpp + smart_buffer.cpp writable_memory.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/data/smart_buffer.cpp b/vespalib/src/vespa/vespalib/data/smart_buffer.cpp new file mode 100644 index 00000000000..401b6729601 --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/smart_buffer.cpp @@ -0,0 +1,68 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "smart_buffer.h" +#include <cassert> + +namespace vespalib { + +void +SmartBuffer::ensure_free(size_t bytes) +{ + if (write_len() >= bytes) { + return; + } + if ((unused() < bytes) || ((unused() * 3) < read_len())) { + size_t new_size = std::max(_data.size() * 2, read_len() + bytes); + alloc::Alloc new_buf(alloc::Alloc::alloc(new_size)); + memcpy(new_buf.get(), read_ptr(), read_len()); + _data.swap(new_buf); + } else { + memmove(_data.get(), read_ptr(), read_len()); + } + _write_pos = read_len(); + _read_pos = 0; +} + +SmartBuffer::SmartBuffer(size_t initial_size) + : _data(alloc::Alloc::alloc(initial_size)), + _read_pos(0), + _write_pos(0) +{ +} + +SmartBuffer::~SmartBuffer() = default; + +Memory +SmartBuffer::obtain() +{ + return Memory(read_ptr(), read_len()); +} + +Input & +SmartBuffer::evict(size_t bytes) +{ + assert(read_len() >= bytes); + _read_pos += bytes; + if (_read_pos == _write_pos) { + _read_pos = 0; + _write_pos = 0; + } + return *this; +} + +WritableMemory +SmartBuffer::reserve(size_t bytes) +{ + ensure_free(bytes); + return WritableMemory(write_ptr(), write_len()); +} + +Output & +SmartBuffer::commit(size_t bytes) +{ + assert(write_len() >= bytes); + _write_pos += bytes; + return *this; +} + +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/smart_buffer.h b/vespalib/src/vespa/vespalib/data/smart_buffer.h new file mode 100644 index 00000000000..f7c4dd05c3e --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/smart_buffer.h @@ -0,0 +1,41 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "input.h" +#include "output.h" +#include <vespa/vespalib/util/alloc.h> + +namespace vespalib { + +/** + * A somewhat smarter buffer compared to SimpleBuffer. Keeps track of + * data in a continuous memory segment. Tries to limit copying of + * data. + **/ +class SmartBuffer : public Input, + public Output +{ +private: + alloc::Alloc _data; + size_t _read_pos; + size_t _write_pos; + + const char *read_ptr() const { return (const char *)(_data.get()) + _read_pos; } + size_t read_len() const { return (_write_pos - _read_pos); } + char *write_ptr() { return (char *)(_data.get()) + _write_pos; } + size_t write_len() const { return (_data.size() - _write_pos); } + size_t unused() const { return (_data.size() - read_len()); } + void ensure_free(size_t bytes); + +public: + SmartBuffer(size_t initial_size); + ~SmartBuffer(); + size_t capacity() const { return _data.size(); } + Memory obtain() override; + Input &evict(size_t bytes) override; + WritableMemory reserve(size_t bytes) override; + Output &commit(size_t bytes) override; +}; + +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp index 38a91456cba..254a9b213ba 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/transport_security_options_reading.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> +#include <vespa/vespalib/data/smart_buffer.h> #include <assert.h> namespace vespalib { @@ -46,14 +47,14 @@ public: class XorCryptoSocket : public CryptoSocket { private: - static constexpr size_t CHUNK_SIZE = 4096; + static constexpr size_t CHUNK_SIZE = 16 * 1024; enum class OP { READ_KEY, WRITE_KEY }; std::vector<OP> _op_stack; - char _my_key; - char _peer_key; - std::vector<char> _readbuf; - std::vector<char> _writebuf; - SocketHandle _socket; + char _my_key; + char _peer_key; + SmartBuffer _input; + SmartBuffer _output; + SocketHandle _socket; bool is_blocked(ssize_t res, int error) const { return ((res < 0) && ((error == EWOULDBLOCK) || (error == EAGAIN))); @@ -95,8 +96,8 @@ public: : std::vector<OP>({OP::READ_KEY, OP::WRITE_KEY})), _my_key(gen_key()), _peer_key(0), - _readbuf(), - _writebuf(), + _input(CHUNK_SIZE * 2), + _output(CHUNK_SIZE * 2), _socket(std::move(socket)) {} int get_fd() const override { return _socket.get(); } HandshakeResult handshake() override { @@ -111,51 +112,57 @@ public: } size_t min_read_buffer_size() const override { return 1; } ssize_t read(char *buf, size_t len) override { - if (_readbuf.empty()) { - _readbuf.resize(CHUNK_SIZE); - ssize_t res = _socket.read(&_readbuf[0], _readbuf.size()); + if (_input.obtain().size == 0) { + auto dst = _input.reserve(CHUNK_SIZE); + ssize_t res = _socket.read(dst.data, dst.size); if (res > 0) { - _readbuf.resize(res); + _input.commit(res); } else { - _readbuf.clear(); - return res; + return res; // eof/error } } return drain(buf, len); } ssize_t drain(char *buf, size_t len) override { - size_t frame = std::min(len, _readbuf.size()); + auto src = _input.obtain(); + size_t frame = std::min(len, src.size); for (size_t i = 0; i < frame; ++i) { - buf[i] = (_readbuf[i] ^ _my_key); + buf[i] = (src.data[i] ^ _my_key); } - _readbuf.erase(_readbuf.begin(), _readbuf.begin() + frame); + _input.evict(frame); return frame; } ssize_t write(const char *buf, size_t len) override { - ssize_t res = flush(); - while (res > 0) { - res = flush(); - } - if (res < 0) { - return res; + if (_output.obtain().size >= CHUNK_SIZE) { + if (flush() < 0) { + return -1; + } + if (_output.obtain().size > 0) { + errno = EWOULDBLOCK; + return -1; + } } size_t frame = std::min(len, CHUNK_SIZE); + auto dst = _output.reserve(frame); for (size_t i = 0; i < frame; ++i) { - _writebuf.push_back(buf[i] ^ _peer_key); + dst.data[i] = (buf[i] ^ _peer_key); } + _output.commit(frame); return frame; } ssize_t flush() override { - if (!_writebuf.empty()) { - ssize_t res = _socket.write(&_writebuf[0], _writebuf.size()); + auto pending = _output.obtain(); + if (pending.size > 0) { + ssize_t res = _socket.write(pending.data, pending.size); if (res > 0) { - _writebuf.erase(_writebuf.begin(), _writebuf.begin() + res); + _output.evict(res); + return 1; // progress } else { assert(res < 0); + return -1; // error } - return res; } - return 0; + return 0; // done } }; diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp index 435f16cc340..494919f449f 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp @@ -34,7 +34,7 @@ CryptoCodecAdapter::hs_try_fill() ssize_t CryptoCodecAdapter::fill_input() { - if (_input.get().size < _codec->min_encode_buffer_size()) { + if (_input.obtain().size < _codec->min_encode_buffer_size()) { auto dst = _input.reserve(_codec->min_encode_buffer_size()); ssize_t res = _socket.read(dst.data, dst.size); if (res > 0) { @@ -85,13 +85,17 @@ CryptoCodecAdapter::handshake() ssize_t CryptoCodecAdapter::read(char *buf, size_t len) { + auto drain_res = drain(buf, len); + if (drain_res != 0) { + return drain_res; + } auto fill_res = fill_input(); if (fill_res <= 0) { return fill_res; } - ssize_t res = drain(buf, len); - if (res != 0) { - return res; + drain_res = drain(buf, len); + if (drain_res != 0) { + return drain_res; } errno = EWOULDBLOCK; return -1; @@ -113,8 +117,14 @@ CryptoCodecAdapter::drain(char *buf, size_t len) ssize_t CryptoCodecAdapter::write(const char *buf, size_t len) { - if (flush_all() < 0) { - return -1; + if (_output.obtain().size >= _codec->min_encode_buffer_size()) { + if (flush() < 0) { + return -1; + } + if (_output.obtain().size > 0) { + errno = EWOULDBLOCK; + return -1; + } } auto dst = _output.reserve(_codec->min_encode_buffer_size()); auto res = _codec->encode(buf, len, dst.data, dst.size); diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h index 6a624ca44f7..f17693cabff 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h @@ -4,7 +4,7 @@ #include <vespa/vespalib/net/crypto_socket.h> #include <vespa/vespalib/net/socket_handle.h> -#include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/data/smart_buffer.h> #include "crypto_codec.h" namespace vespalib::net::tls { @@ -19,8 +19,8 @@ namespace vespalib::net::tls { class CryptoCodecAdapter : public CryptoSocket { private: - SimpleBuffer _input; - SimpleBuffer _output; + SmartBuffer _input; + SmartBuffer _output; SocketHandle _socket; std::unique_ptr<CryptoCodec> _codec; @@ -33,7 +33,7 @@ private: ssize_t flush_all(); // -1/0 -> error/ok public: CryptoCodecAdapter(SocketHandle socket, std::unique_ptr<CryptoCodec> codec) - : _socket(std::move(socket)), _codec(std::move(codec)) {} + : _input(64 * 1024), _output(64 * 1024), _socket(std::move(socket)), _codec(std::move(codec)) {} int get_fd() const override { return _socket.get(); } HandshakeResult handshake() override; size_t min_read_buffer_size() const override { return _codec->min_decode_buffer_size(); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index d71660a990f..23fa3cccad2 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -3,15 +3,13 @@ package com.yahoo.vespa.curator; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.transaction.Mutex; -import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.locks.InterProcessLock; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; import java.time.Duration; import java.util.concurrent.TimeUnit; /** - * A cluster-wide reentrant mutex which is released on (the last symmetric) close + * A cluster-wide re-entrant mutex which is released on (the last symmetric) close * * @author bratseth */ @@ -20,13 +18,6 @@ public class Lock implements Mutex { private final InterProcessLock mutex; private final String lockPath; - /** @deprecated pass a Curator instance instead */ - @Deprecated - public Lock(String lockPath, CuratorFramework curator) { - this.lockPath = lockPath; - mutex = new InterProcessMutex(curator, lockPath); - } - public Lock(String lockPath, Curator curator) { this.lockPath = lockPath; mutex = curator.createMutex(lockPath); |