diff options
213 files changed, 2980 insertions, 4255 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index 0eeeae457b6..a117d283146 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -2,13 +2,16 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.Zone; -import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; import com.yahoo.log.LogLevel; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.MutableX509KeyManager; +import com.yahoo.security.tls.PeerAuthentication; +import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.Identity; @@ -17,12 +20,11 @@ import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.utils.SiaUtils; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import org.eclipse.jetty.util.ssl.SslContextFactory; +import javax.net.ssl.SSLContext; import java.net.URI; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.GeneralSecurityException; import java.security.KeyPair; import java.security.KeyStore; import java.security.PrivateKey; @@ -42,14 +44,15 @@ import java.util.logging.Logger; * * @author bjorncs */ -public class ConfigserverSslContextFactoryProvider extends AbstractComponent implements SslContextFactoryProvider { +public class ConfigserverSslContextFactoryProvider extends TlsContextBasedProvider { private static final String CERTIFICATE_ALIAS = "athenz"; private static final Duration EXPIRATION_MARGIN = Duration.ofHours(6); private static final Path VESPA_SIA_DIRECTORY = Paths.get(Defaults.getDefaults().underVespaHome("var/vespa/sia")); private static final Logger log = Logger.getLogger(ConfigserverSslContextFactoryProvider.class.getName()); - private final SslContextFactory sslContextFactory; + private final TlsContext tlsContext; + private final MutableX509KeyManager keyManager = new MutableX509KeyManager(); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(runnable -> new Thread(runnable, "configserver-ssl-context-factory-provider")); private final ZtsClient ztsClient; @@ -60,8 +63,7 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp @Inject public ConfigserverSslContextFactoryProvider(ServiceIdentityProvider bootstrapIdentity, KeyProvider keyProvider, - AthenzProviderServiceConfig config, - Zone zone) { + AthenzProviderServiceConfig config) { this.athenzProviderServiceConfig = config; this.ztsClient = new DefaultZtsClient(URI.create(athenzProviderServiceConfig.ztsUrl()), bootstrapIdentity); this.keyProvider = keyProvider; @@ -69,25 +71,20 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp Duration updatePeriod = Duration.ofDays(config.updatePeriodDays()); Path trustStoreFile = Paths.get(config.athenzCaTrustStore()); - this.sslContextFactory = initializeSslContextFactory(keyProvider, trustStoreFile, updatePeriod, configserverIdentity, ztsClient, athenzProviderServiceConfig); - scheduler.scheduleAtFixedRate(new KeystoreUpdater(sslContextFactory), + this.tlsContext = createTlsContext(keyProvider, keyManager, trustStoreFile, updatePeriod, configserverIdentity, ztsClient, athenzProviderServiceConfig); + scheduler.scheduleAtFixedRate(new KeystoreUpdater(keyManager), updatePeriod.toDays()/*initial delay*/, updatePeriod.toDays(), TimeUnit.DAYS); } @Override - public SslContextFactory getInstance(String containerId, int port) { - return sslContextFactory; + protected TlsContext getTlsContext(String containerId, int port) { + return tlsContext; } Instant getCertificateNotAfter() { - try { - X509Certificate certificate = (X509Certificate) sslContextFactory.getKeyStore().getCertificate(CERTIFICATE_ALIAS); - return certificate.getNotAfter().toInstant(); - } catch (GeneralSecurityException e) { - throw new IllegalStateException("Unable to find configserver certificate from keystore: " + e.getMessage(), e); - } + return keyManager.currentManager().getCertificateChain(CERTIFICATE_ALIAS)[0].getNotAfter().toInstant(); } @Override @@ -96,38 +93,28 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp scheduler.shutdownNow(); scheduler.awaitTermination(30, TimeUnit.SECONDS); ztsClient.close(); + super.deconstruct(); } catch (InterruptedException e) { throw new RuntimeException("Failed to shutdown Athenz certificate updater on time", e); } } - private static SslContextFactory initializeSslContextFactory(KeyProvider keyProvider, - Path trustStoreFile, - Duration updatePeriod, - AthenzService configserverIdentity, - ZtsClient ztsClient, - AthenzProviderServiceConfig zoneConfig) { - - // TODO Use DefaultTlsContext to configure SslContextFactory (ensure that cipher/protocol configuration is same across all TLS endpoints) - - SslContextFactory.Server factory = new SslContextFactory.Server(); - - factory.setWantClientAuth(true); - - KeyStore trustStore = - KeyStoreBuilder.withType(KeyStoreType.JKS) - .fromFile(trustStoreFile) - .build(); - factory.setTrustStore(trustStore); - + private static TlsContext createTlsContext(KeyProvider keyProvider, + MutableX509KeyManager keyManager, + Path trustStoreFile, + Duration updatePeriod, + AthenzService configserverIdentity, + ZtsClient ztsClient, + AthenzProviderServiceConfig zoneConfig) { KeyStore keyStore = tryReadKeystoreFile(configserverIdentity, updatePeriod) .orElseGet(() -> updateKeystore(configserverIdentity, generateKeystorePassword(), keyProvider, ztsClient, zoneConfig)); - factory.setKeyStore(keyStore); - factory.setKeyStorePassword(""); - factory.setExcludeProtocols("TLSv1.3"); // TLSv1.3 is broken is multiple OpenJDK 11 versions - factory.setEndpointIdentificationAlgorithm(null); // disable https hostname verification of clients (must be disabled when using Athenz x509 certificates) - return factory; + keyManager.updateKeystore(keyStore, new char[0]); + SSLContext sslContext = new SslContextBuilder() + .withTrustStore(trustStoreFile, KeyStoreType.JKS) + .withKeyManager(keyManager) + .build(); + return new DefaultTlsContext(sslContext, PeerAuthentication.WANT); } private static Optional<KeyStore> tryReadKeystoreFile(AthenzService configserverIdentity, Duration updatePeriod) { @@ -171,10 +158,10 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp } private class KeystoreUpdater implements Runnable { - final SslContextFactory sslContextFactory; + final MutableX509KeyManager keyManager; - KeystoreUpdater(SslContextFactory sslContextFactory) { - this.sslContextFactory = sslContextFactory; + KeystoreUpdater(MutableX509KeyManager keyManager) { + this.keyManager = keyManager; } @Override @@ -183,10 +170,7 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp log.log(LogLevel.INFO, "Updating configserver provider certificate from ZTS"); char[] keystorePwd = generateKeystorePassword(); KeyStore keyStore = updateKeystore(configserverIdentity, keystorePwd, keyProvider, ztsClient, athenzProviderServiceConfig); - sslContextFactory.reload(scf -> { - scf.setKeyStore(keyStore); - scf.setKeyStorePassword(new String(keystorePwd)); - }); + keyManager.updateKeystore(keyStore, keystorePwd); log.log(LogLevel.INFO, "Certificate successfully updated"); } catch (Throwable t) { log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + t.getMessage(), t); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index ba35243c14d..364184331a8 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -661,6 +661,12 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } private boolean broadcastClusterStateToEligibleNodes() { + // If there's a pending DB store we have not yet been able to store the + // current state bundle to ZK and must therefore _not_ allow it to be published. + if (database.hasPendingClusterStateMetaDataStore()) { + log.log(LogLevel.DEBUG, "Can't publish current cluster state as it has one or more pending ZooKeeper stores"); + return false; + } boolean sentAny = false; // Give nodes a fair chance to respond first time to state gathering requests, so we don't // disturb system when we take over. Allow anyways if we have states from all nodes. diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java index f2b1b523aba..f30b86130c2 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java @@ -352,6 +352,15 @@ public class DatabaseHandler { doNextZooKeeperTask(context); } + // TODO should we expand this to cover _any_ pending ZK write? + public boolean hasPendingClusterStateMetaDataStore() { + synchronized (databaseMonitor) { + return ((zooKeeperAddress != null) && + ((pendingStore.clusterStateBundle != null) || + (pendingStore.lastSystemStateVersion != null))); + } + } + public ClusterStateBundle getLatestClusterStateBundle() throws InterruptedException { log.log(LogLevel.DEBUG, () -> String.format("Fleetcontroller %d: Retrieving latest cluster state bundle from ZooKeeper", nodeIndex)); synchronized (databaseMonitor) { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java index ca8eadd8d1f..158fbfb175f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java @@ -13,7 +13,7 @@ import java.util.stream.Collectors; * endpoint (endpointId) and the name of the container cluster that the endpoint * should point to. * - * If the endpointId is not set, it will default to the same as the containerId. + * If the endpoint is not set it will default to the string "default". * * @author ogronnesby */ diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 650f68591b6..72a806bb7be 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -161,7 +162,7 @@ public class DeploymentSpecXmlReader { final var endpointsElement = XML.getChild(root, endpointsTag); if (endpointsElement == null) { return Collections.emptyList(); } - final var endpoints = new ArrayList<Endpoint>(); + final var endpoints = new LinkedHashMap<String, Endpoint>(); for (var endpointElement : XML.getChildren(endpointsElement, endpointTag)) { final Optional<String> rotationId = stringAttribute("id", endpointElement); @@ -184,13 +185,13 @@ public class DeploymentSpecXmlReader { } var endpoint = new Endpoint(rotationId, containerId.get(), regions); - if (endpoints.contains(endpoint)) { - throw new IllegalArgumentException("Duplicate 'endpoint' in 'endpoints' tag"); + if (endpoints.containsKey(endpoint.endpointId())) { + throw new IllegalArgumentException("Duplicate attribute 'id' on 'endpoint': " + endpoint.endpointId()); } - endpoints.add(endpoint); + endpoints.put(endpoint.endpointId(), endpoint); } - return endpoints; + return List.copyOf(endpoints.values()); } /** diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 9d7ae9759c3..3b4bc1c2c5c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -57,8 +57,6 @@ public interface ModelContext { boolean useFdispatchByDefault(); boolean dispatchWithProtobuf(); boolean useAdaptiveDispatch(); - // TODO: Remove when 7.61 is the oldest model in use - default boolean enableMetricsProxyContainer() { return false; } // TODO: Remove temporary default implementation default Optional<TlsSecrets> tlsSecrets() { return Optional.empty(); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index a7c0ebd4a07..a871da20669 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -28,12 +28,19 @@ public class ExactMatch extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { - Matching.Type matching = field.getMatching().getType(); - if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { - implementExactMatch(field, search); - } else if (field.getMatching().getExactMatchTerminator() != null) { - warn(search, field, "exact-terminator requires 'exact' matching to have any effect."); - } + processField(field, search); + } + } + + private void processField(SDField field, Search search) { + Matching.Type matching = field.getMatching().getType(); + if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { + implementExactMatch(field, search); + } else if (field.getMatching().getExactMatchTerminator() != null) { + warn(search, field, "exact-terminator requires 'exact' matching to have any effect."); + } + for (var structField : field.getStructFields()) { + processField(structField, search); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index e9db64f8e4b..73a401e6a2a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -11,6 +11,7 @@ import com.yahoo.container.BundlesConfig; import com.yahoo.jdisc.http.ServletPathsConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ConfigProducerGroup; import com.yahoo.vespa.model.container.component.Servlet; @@ -19,6 +20,7 @@ import com.yahoo.vespa.model.container.jersey.RestApi; import com.yahoo.vespa.model.utils.FileSender; import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; @@ -60,6 +62,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.SecretStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.DeprecatedSecretStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); + addTestrunnerComponentsIfTester(deployState); } @Override @@ -86,6 +89,11 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } } + private void addTestrunnerComponentsIfTester(DeployState deployState) { + if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) + addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-testrunner-components-jar-with-dependencies.jar"))); + } + public void setModelEvaluation(ContainerModelEvaluation modelEvaluation) { this.modelEvaluation = modelEvaluation; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 7a6b1064b24..47adac637ee 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -37,7 +37,6 @@ import com.yahoo.search.config.QrStartConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; -import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.PortsMeta; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.admin.monitoring.Monitoring; @@ -64,7 +63,6 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -192,7 +190,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> addSimpleComponent("com.yahoo.container.handler.VipStatus"); addSimpleComponent(com.yahoo.container.handler.ClustersStatus.class.getName()); addJaxProviders(); - addTestrunnerComponentsIfTester(deployState); } public void setZone(Zone zone) { @@ -207,11 +204,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> addVipHandler(); } - private void addTestrunnerComponentsIfTester(DeployState deployState) { - if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) - addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-testrunner-components-jar-with-dependencies.jar"))); - } - public final void addDefaultHandlersExceptStatus() { addDefaultRootHandler(); addMetricStateHandler(); diff --git a/config-model/src/test/derived/exactmatch/exactmatch.sd b/config-model/src/test/derived/exactmatch/exactmatch.sd index d5104fbf36d..2f1d54c970a 100644 --- a/config-model/src/test/derived/exactmatch/exactmatch.sd +++ b/config-model/src/test/derived/exactmatch/exactmatch.sd @@ -3,6 +3,11 @@ search exactmatch { document exactmatch { + struct elem { + field name type string {} + field weight type int {} + } + field tag type string { indexing: summary | index match: exact @@ -16,6 +21,32 @@ search exactmatch { } } + field string_map type map<string, string> { + indexing: summary + struct-field key { + indexing: attribute + match { + exact + exact-terminator: "*!!!*" + } + } + } + + field elem_map type map<string, elem> { + indexing: summary + struct-field value.name { + indexing: attribute + match: exact + } + } + + field elem_array type array<elem> { + indexing: summary + struct-field name { + indexing: attribute + match: exact + } + } } } diff --git a/config-model/src/test/derived/exactmatch/ilscripts.cfg b/config-model/src/test/derived/exactmatch/ilscripts.cfg index 38c0978474f..5595f954c4a 100644 --- a/config-model/src/test/derived/exactmatch/ilscripts.cfg +++ b/config-model/src/test/derived/exactmatch/ilscripts.cfg @@ -3,5 +3,11 @@ fieldmatchmaxlength 1000000 ilscript[].doctype "exactmatch" ilscript[].docfield[] "tag" ilscript[].docfield[] "screweduserids" +ilscript[].docfield[] "string_map" +ilscript[].docfield[] "elem_map" +ilscript[].docfield[] "elem_array" ilscript[].content[] "clear_state | guard { input tag | exact | summary tag | index tag; }" ilscript[].content[] "clear_state | guard { input screweduserids | exact | index screweduserids | summary screweduserids | attribute screweduserids; }" +ilscript[].content[] "input elem_array | passthrough elem_array" +ilscript[].content[] "input elem_map | passthrough elem_map" +ilscript[].content[] "input string_map | passthrough string_map" diff --git a/config-model/src/test/derived/exactmatch/index-info.cfg b/config-model/src/test/derived/exactmatch/index-info.cfg index a17ff68642e..a4a193b1fcd 100644 --- a/config-model/src/test/derived/exactmatch/index-info.cfg +++ b/config-model/src/test/derived/exactmatch/index-info.cfg @@ -15,3 +15,47 @@ indexinfo[].command[].indexname "screweduserids" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "screweduserids" indexinfo[].command[].command "exact *!!!*" +indexinfo[].command[].indexname "string_map.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "string_map.key" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "string_map.key" +indexinfo[].command[].command "exact *!!!*" +indexinfo[].command[].indexname "string_map.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "string_map" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "string_map" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_map.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_map.value.name" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_map.value.name" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "elem_map.value.name" +indexinfo[].command[].command "exact @@" +indexinfo[].command[].indexname "elem_map.value.weight" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_map.value.weight" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "elem_map.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_map" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_map" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "exact @@" +indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "elem_array" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_array" +indexinfo[].command[].command "multivalue" diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index db3b787f9f9..5ffc7293742 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -1,10 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import com.yahoo.jrt.*; +import com.yahoo.jrt.Acceptor; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.ListenFailedException; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringArray; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.TargetWatcher; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.config.*; import com.yahoo.vespa.config.ErrorCode; +import com.yahoo.vespa.config.JRTMethods; +import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.protocol.JRTConfigRequestFactory; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; @@ -12,6 +23,9 @@ import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import java.util.Arrays; import java.util.Iterator; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -28,6 +42,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer private final Spec spec; private final Supervisor supervisor; private final ProxyServer proxyServer; + private final ExecutorService rpcExecutor = Executors.newFixedThreadPool(8); ConfigProxyRpcServer(ProxyServer proxyServer, Supervisor supervisor, Spec spec) { this.proxyServer = proxyServer; @@ -50,6 +65,12 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer void shutdown() { supervisor.transport().shutdown(); + try { + rpcExecutor.shutdownNow(); + rpcExecutor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } Spec getSpec() { @@ -109,12 +130,16 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer * @param req a Request */ private void getConfigV3(Request req) { - log.log(LogLevel.SPAM, () -> "getConfigV3"); - JRTServerConfigRequest request = JRTServerConfigRequestV3.createFromRequest(req); - if (isProtocolVersionSupported(request)) { - preHandle(req); - getConfigImpl(request); - } + dispatchRpcRequest(req, () -> { + JRTServerConfigRequest request = JRTServerConfigRequestV3.createFromRequest(req); + if (isProtocolVersionSupported(request)) { + proxyServer.getStatistics().incRpcRequests(); + req.target().addWatcher(this); + getConfigImpl(request); + return; + } + req.returnRequest(); + }); } /** @@ -122,8 +147,11 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer * * @param req a Request */ - void ping(Request req) { - req.returnValues().add(new Int32Value(0)); + private void ping(Request req) { + dispatchRpcRequest(req, () -> { + req.returnValues().add(new Int32Value(0)); + req.returnRequest(); + }); } /** @@ -131,81 +159,120 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer * * @param req a Request */ - void printStatistics(Request req) { - StringBuilder sb = new StringBuilder(); - sb.append("\nDelayed responses queue size: "); - sb.append(proxyServer.delayedResponses.size()); - sb.append("\nContents: "); - for (DelayedResponse delayed : proxyServer.delayedResponses.responses()) { - sb.append(delayed.getRequest().toString()).append("\n"); - } - - req.returnValues().add(new StringValue(sb.toString())); - } + private void printStatistics(Request req) { + dispatchRpcRequest(req, () -> { + StringBuilder sb = new StringBuilder(); + sb.append("\nDelayed responses queue size: "); + sb.append(proxyServer.delayedResponses.size()); + sb.append("\nContents: "); + for (DelayedResponse delayed : proxyServer.delayedResponses.responses()) { + sb.append(delayed.getRequest().toString()).append("\n"); + } - void listCachedConfig(Request req) { - listCachedConfig(req, false); + req.returnValues().add(new StringValue(sb.toString())); + req.returnRequest(); + }); } - void listCachedConfigFull(Request req) { - listCachedConfig(req, true); + private void listCachedConfig(Request req) { + dispatchRpcRequest(req, () -> listCachedConfig(req, false)); } - void listSourceConnections(Request req) { - String[] ret = new String[2]; - ret[0] = "Current source: " + proxyServer.getActiveSourceConnection(); - ret[1] = "All sources:\n" + printSourceConnections(); - req.returnValues().add(new StringArray(ret)); + private void listCachedConfigFull(Request req) { + dispatchRpcRequest(req, () -> listCachedConfig(req, true)); } - void updateSources(Request req) { - String sources = req.parameters().get(0).asString(); - String ret; - System.out.println(proxyServer.getMode()); - if (proxyServer.getMode().requiresConfigSource()) { - proxyServer.updateSourceConnections(Arrays.asList(sources.split(","))); - ret = "Updated config sources to: " + sources; - } else { - ret = "Cannot update sources when in '" + proxyServer.getMode().name() + "' mode"; - } - req.returnValues().add(new StringValue(ret)); + private void listSourceConnections(Request req) { + dispatchRpcRequest(req, () -> { + String[] ret = new String[2]; + ret[0] = "Current source: " + proxyServer.getActiveSourceConnection(); + ret[1] = "All sources:\n" + printSourceConnections(); + req.returnValues().add(new StringArray(ret)); + req.returnRequest(); + }); } - void invalidateCache(Request req) { - proxyServer.getMemoryCache().clear(); - String[] s = new String[2]; - s[0] = "0"; - s[1] = "success"; - req.returnValues().add(new StringArray(s)); + private void updateSources(Request req) { + dispatchRpcRequest(req, () -> { + String sources = req.parameters().get(0).asString(); + String ret; + System.out.println(proxyServer.getMode()); + if (proxyServer.getMode().requiresConfigSource()) { + proxyServer.updateSourceConnections(Arrays.asList(sources.split(","))); + ret = "Updated config sources to: " + sources; + } else { + ret = "Cannot update sources when in '" + proxyServer.getMode().name() + "' mode"; + } + req.returnValues().add(new StringValue(ret)); + req.returnRequest(); + }); } - void setMode(Request req) { - String suppliedMode = req.parameters().get(0).asString(); - log.log(LogLevel.DEBUG, () -> "Supplied mode=" + suppliedMode); - String[] s = new String[2]; - if (Mode.validModeName(suppliedMode.toLowerCase())) { - proxyServer.setMode(suppliedMode); + private void invalidateCache(Request req) { + dispatchRpcRequest(req, () -> { + proxyServer.getMemoryCache().clear(); + String[] s = new String[2]; s[0] = "0"; s[1] = "success"; - } else { - s[0] = "1"; - s[1] = "Could not set mode to '" + suppliedMode + "'. Legal modes are '" + Mode.modes() + "'"; - } + req.returnValues().add(new StringArray(s)); + req.returnRequest(); + }); + } + + private void setMode(Request req) { + dispatchRpcRequest(req, () -> { + String suppliedMode = req.parameters().get(0).asString(); + log.log(LogLevel.DEBUG, () -> "Supplied mode=" + suppliedMode); + String[] s = new String[2]; + if (Mode.validModeName(suppliedMode.toLowerCase())) { + proxyServer.setMode(suppliedMode); + s[0] = "0"; + s[1] = "success"; + } else { + s[0] = "1"; + s[1] = "Could not set mode to '" + suppliedMode + "'. Legal modes are '" + Mode.modes() + "'"; + } - req.returnValues().add(new StringArray(s)); + req.returnValues().add(new StringArray(s)); + req.returnRequest(); + }); } - void getMode(Request req) { - req.returnValues().add(new StringValue(proxyServer.getMode().name())); + private void getMode(Request req) { + dispatchRpcRequest(req, () -> { + req.returnValues().add(new StringValue(proxyServer.getMode().name())); + req.returnRequest(); + }); } - void dumpCache(Request req) { - final MemoryCache memoryCache = proxyServer.getMemoryCache(); - req.returnValues().add(new StringValue(memoryCache.dumpCacheToDisk(req.parameters().get(0).asString(), memoryCache))); + private void dumpCache(Request req) { + dispatchRpcRequest(req, () -> { + final MemoryCache memoryCache = proxyServer.getMemoryCache(); + req.returnValues().add(new StringValue(memoryCache.dumpCacheToDisk(req.parameters().get(0).asString(), memoryCache))); + req.returnRequest(); + }); } //---------------------------------------------------- + private void dispatchRpcRequest(Request request, Runnable handler) { + request.detach(); + log.log(LogLevel.SPAM, () -> String.format("Dispatching RPC request %s", requestLogId(request))); + rpcExecutor.execute(() -> { + try { + log.log(LogLevel.SPAM, () -> String.format("Executing RPC request %s.", requestLogId(request))); + handler.run(); + } catch (Exception e) { + log.log(LogLevel.WARNING, + String.format("Exception thrown during execution of RPC request %s: %s", requestLogId(request), e.getMessage()), e); + } + }); + } + + private String requestLogId(Request request) { + return String.format("%s/%08X", request.methodName(), request.hashCode()); + } + private boolean isProtocolVersionSupported(JRTServerConfigRequest request) { Set<Long> supportedProtocolVersions = JRTConfigRequestFactory.supportedProtocolVersions(); if (supportedProtocolVersions.contains(request.getProtocolVersion())) { @@ -219,12 +286,6 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer return false; } - private void preHandle(Request req) { - proxyServer.getStatistics().incRpcRequests(); - req.detach(); - req.target().addWatcher(this); - } - /** * Handles all versions of "getConfig" requests. * @@ -262,7 +323,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer return sb.toString(); } - final void listCachedConfig(Request req, boolean full) { + private void listCachedConfig(Request req, boolean full) { String[] ret; MemoryCache cache = proxyServer.getMemoryCache(); ret = new String[cache.size()]; @@ -287,6 +348,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer } Arrays.sort(ret); req.returnValues().add(new StringArray(ret)); + req.returnRequest(); } /** diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 40526641855..55e546072fc 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -9,12 +9,10 @@ import com.yahoo.jrt.Transport; import com.yahoo.log.LogLevel; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; -import com.yahoo.vespa.config.JRTConnectionPool; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; -import com.yahoo.vespa.filedistribution.FileDistributionRpcServer; -import com.yahoo.vespa.filedistribution.FileDownloader; +import com.yahoo.vespa.config.proxy.filedistribution.FileDistributionAndUrlDownload; import com.yahoo.yolean.system.CatchSignals; import java.util.List; @@ -37,6 +35,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; public class ProxyServer implements Runnable { private static final int DEFAULT_RPC_PORT = 19090; + private static final int JRT_TRANSPORT_THREADS = 4; static final String DEFAULT_PROXY_CONFIG_SOURCES = "tcp/localhost:19070"; final static Logger log = Logger.getLogger(ProxyServer.class.getName()); @@ -44,7 +43,7 @@ public class ProxyServer implements Runnable { // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory()); - private final Supervisor supervisor = new Supervisor(new Transport()); + private final Supervisor supervisor = new Supervisor(new Transport(JRT_TRANSPORT_THREADS)); private final ClientUpdater clientUpdater; private ScheduledFuture<?> delayedResponseScheduler; @@ -60,6 +59,7 @@ public class ProxyServer implements Runnable { private static final double timingValuesRatio = 0.8; private final static TimingValues defaultTimingValues; private final boolean delayedResponseHandling; + private final FileDistributionAndUrlDownload fileDistributionAndUrlDownload; private volatile Mode mode = new Mode(DEFAULT); @@ -87,8 +87,7 @@ public class ProxyServer implements Runnable { this.rpcServer = createRpcServer(spec); clientUpdater = new ClientUpdater(rpcServer, statistics, delayedResponses); this.configClient = createClient(clientUpdater, delayedResponses, source, timingValues, memoryCache, configClient); - new FileDistributionRpcServer(supervisor, new FileDownloader(new JRTConnectionPool(source))); - new UrlDownloadRpcServer(supervisor); + this.fileDistributionAndUrlDownload = new FileDistributionAndUrlDownload(supervisor, source); } static ProxyServer createTestServer(ConfigSourceSet source) { @@ -266,6 +265,7 @@ public class ProxyServer implements Runnable { if (statistics != null) { statistics.stop(); } + fileDistributionAndUrlDownload.close(); } MemoryCache getMemoryCache() { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java new file mode 100644 index 00000000000..0b7de6ed562 --- /dev/null +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java @@ -0,0 +1,29 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.proxy.filedistribution; + +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.jrt.Supervisor; +import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.filedistribution.FileDownloader; + +/** + * Keeps track of file distribution and url download rpc servers. + * + * @author hmusum + */ +public class FileDistributionAndUrlDownload { + + private final FileDistributionRpcServer fileDistributionRpcServer; + private final UrlDownloadRpcServer urlDownloadRpcServer; + + public FileDistributionAndUrlDownload(Supervisor supervisor, ConfigSourceSet source) { + fileDistributionRpcServer = new FileDistributionRpcServer(supervisor, new FileDownloader(new JRTConnectionPool(source))); + urlDownloadRpcServer = new UrlDownloadRpcServer(supervisor); + } + + public void close() { + fileDistributionRpcServer.close(); + urlDownloadRpcServer.close(); + } + +} diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java index d27d7422beb..33a8ed405a9 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.filedistribution; +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.proxy.filedistribution; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.FileReference; @@ -11,6 +11,8 @@ import com.yahoo.jrt.StringArray; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.filedistribution.FileDownloader; +import com.yahoo.vespa.filedistribution.FileReferenceDownload; import java.io.File; import java.util.Arrays; @@ -18,6 +20,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -41,6 +44,15 @@ public class FileDistributionRpcServer { declareFileDistributionMethods(); } + public void close() { + rpcDownloadExecutor.shutdownNow(); + try { + rpcDownloadExecutor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + private void declareFileDistributionMethods() { // Legacy method, needs to be the same name as used in filedistributor supervisor.addMethod(new Method("waitFor", "s", "s", this::getFile) diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java index 711c43340cb..9d89f1d10b2 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java @@ -1,5 +1,5 @@ -// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.proxy; +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.proxy.filedistribution; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.jrt.Method; @@ -26,6 +26,7 @@ import java.nio.channels.ReadableByteChannel; import java.nio.file.Files; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import static com.yahoo.vespa.config.UrlDownloader.DOES_NOT_EXIST; @@ -37,7 +38,7 @@ import static com.yahoo.vespa.config.UrlDownloader.INTERNAL_ERROR; * * @author lesters */ -public class UrlDownloadRpcServer { +class UrlDownloadRpcServer { private final static Logger log = Logger.getLogger(UrlDownloadRpcServer.class.getName()); private static final String CONTENTS_FILE_NAME = "contents"; @@ -45,7 +46,7 @@ public class UrlDownloadRpcServer { private final File downloadBaseDir; private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), - new DaemonThreadFactory("Rpc download executor")); + new DaemonThreadFactory("Rpc URL download executor")); UrlDownloadRpcServer(Supervisor supervisor) { supervisor.addMethod(new Method("url.waitFor", "s", "s", this::download) @@ -55,6 +56,15 @@ public class UrlDownloadRpcServer { downloadBaseDir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/download")); } + void close() { + rpcDownloadExecutor.shutdownNow(); + try { + rpcDownloadExecutor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + private void download(Request req) { req.detach(); rpcDownloadExecutor.execute(() -> downloadFile(req)); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java index f32bb2ac024..48456d8ac23 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java @@ -2,40 +2,46 @@ package com.yahoo.vespa.config.proxy; import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.jrt.Acceptor; +import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.Request; import com.yahoo.jrt.Spec; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.vespa.config.RawConfig; import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.time.Duration; + import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; /** * @author hmusum - * @since 5.1.9 + * @author bjorncs */ public class ConfigProxyRpcServerTest { private static final String hostname = "localhost"; private static final int port = 12345; private static final String address = "tcp/" + hostname + ":" + port; - private ProxyServer proxyServer; - private ConfigProxyRpcServer rpcServer; + private TestServer server; + private TestClient client; @Before - public void setup() { - proxyServer = ProxyServer.createTestServer(new ConfigSourceSet(address)); - rpcServer = new ConfigProxyRpcServer(proxyServer, new Supervisor(new Transport()), null); + public void setup() throws ListenFailedException { + server = new TestServer(); + client = new TestClient(server.listenPort()); } @After public void teardown() { - rpcServer.shutdown(); + client.close(); + server.close(); } @Test @@ -52,7 +58,7 @@ public class ConfigProxyRpcServerTest { @Test public void testRpcMethodPing() { Request req = new Request("ping"); - rpcServer.ping(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); @@ -65,7 +71,7 @@ public class ConfigProxyRpcServerTest { @Test public void testRpcMethodListCachedConfig() { Request req = new Request("listCachedConfig"); - rpcServer.listCachedConfig(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); String[] ret = req.returnValues().get(0).asStringArray(); @@ -73,9 +79,9 @@ public class ConfigProxyRpcServerTest { assertThat(ret.length, is(0)); final RawConfig config = ProxyServerTest.fooConfig; - proxyServer.getMemoryCache().update(config); + server.proxyServer().getMemoryCache().update(config); req = new Request("listCachedConfig"); - rpcServer.listCachedConfig(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); ret = req.returnValues().get(0).asStringArray(); @@ -92,7 +98,7 @@ public class ConfigProxyRpcServerTest { @Test public void testRpcMethodListCachedConfigFull() { Request req = new Request("listCachedConfigFull"); - rpcServer.listCachedConfigFull(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); @@ -100,9 +106,9 @@ public class ConfigProxyRpcServerTest { assertThat(ret.length, is(0)); final RawConfig config = ProxyServerTest.fooConfig; - proxyServer.getMemoryCache().update(config); + server.proxyServer().getMemoryCache().update(config); req = new Request("listCachedConfigFull"); - rpcServer.listCachedConfigFull(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); ret = req.returnValues().get(0).asStringArray(); assertThat(ret.length, is(1)); @@ -119,7 +125,7 @@ public class ConfigProxyRpcServerTest { @Test public void testRpcMethodListSourceConnections() { Request req = new Request("listSourceConnections"); - rpcServer.listSourceConnections(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); @@ -135,7 +141,7 @@ public class ConfigProxyRpcServerTest { @Test public void testRpcMethodPrintStatistics() { Request req = new Request("printStatistics"); - rpcServer.printStatistics(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); assertThat(req.returnValues().get(0).asString(), is("\n" + @@ -149,7 +155,7 @@ public class ConfigProxyRpcServerTest { @Test public void testRpcMethodInvalidateCache() { Request req = new Request("invalidateCache"); - rpcServer.invalidateCache(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); @@ -165,7 +171,7 @@ public class ConfigProxyRpcServerTest { @Test public void testRpcMethodGetModeAndSetMode() { Request req = new Request("getMode"); - rpcServer.getMode(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); assertThat(req.returnValues().get(0).asString(), is("default")); @@ -173,17 +179,17 @@ public class ConfigProxyRpcServerTest { req = new Request("setMode"); String mode = "memorycache"; req.parameters().add(new StringValue(mode)); - rpcServer.setMode(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); String[] ret = req.returnValues().get(0).asStringArray(); assertThat(ret.length, is(2)); assertThat(ret[0], is("0")); assertThat(ret[1], is("success")); - assertThat(proxyServer.getMode().name(), is(mode)); + assertThat(server.proxyServer().getMode().name(), is(mode)); req = new Request("getMode"); - rpcServer.getMode(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); assertThat(req.returnValues().get(0).asString(), is(mode)); @@ -192,14 +198,14 @@ public class ConfigProxyRpcServerTest { String oldMode = mode; mode = "invalid"; req.parameters().add(new StringValue(mode)); - rpcServer.setMode(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); ret = req.returnValues().get(0).asStringArray(); assertThat(ret.length, is(2)); assertThat(ret[0], is("1")); assertThat(ret[1], is("Could not set mode to '" + mode + "'. Legal modes are '" + Mode.modes() + "'")); - assertThat(proxyServer.getMode().name(), is(oldMode)); + assertThat(server.proxyServer().getMode().name(), is(oldMode)); } /** @@ -211,17 +217,17 @@ public class ConfigProxyRpcServerTest { String spec1 = "tcp/a:19070"; String spec2 = "tcp/b:19070"; req.parameters().add(new StringValue(spec1 + "," + spec2)); - rpcServer.updateSources(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); assertThat(req.returnValues().get(0).asString(), is("Updated config sources to: " + spec1 + "," + spec2)); - proxyServer.setMode(Mode.ModeName.MEMORYCACHE.name()); + server.proxyServer().setMode(Mode.ModeName.MEMORYCACHE.name()); req = new Request("updateSources"); req.parameters().add(new StringValue(spec1 + "," + spec2)); - rpcServer.updateSources(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); assertThat(req.returnValues().get(0).asString(), is("Cannot update sources when in '" + Mode.ModeName.MEMORYCACHE.name().toLowerCase() + "' mode")); @@ -245,10 +251,59 @@ public class ConfigProxyRpcServerTest { Request req = new Request("dumpCache"); String path = "/tmp"; req.parameters().add(new StringValue(path)); - rpcServer.dumpCache(req); + client.invoke(req); assertFalse(req.errorMessage(), req.isError()); assertThat(req.returnValues().size(), is(1)); assertThat(req.returnValues().get(0).asString(), is("success")); } + private static class TestServer implements AutoCloseable { + + private static final Spec SPEC = new Spec(0); + + private final ProxyServer proxyServer = ProxyServer.createTestServer(new ConfigSourceSet(address)); + private final Supervisor supervisor = new Supervisor(new Transport()); + private final ConfigProxyRpcServer rpcServer = new ConfigProxyRpcServer(proxyServer, supervisor, SPEC); + private final Acceptor acceptor; + + TestServer() throws ListenFailedException { + acceptor = supervisor.listen(SPEC); + } + + ProxyServer proxyServer() { + return proxyServer; + } + + int listenPort() { + return acceptor.port(); + } + + @Override + public void close() { + acceptor.shutdown().join(); + supervisor.transport().shutdown().join(); + rpcServer.shutdown(); + } + } + + private static class TestClient implements AutoCloseable { + + private final Supervisor supervisor; + private final Target target; + + TestClient(int rpcPort) { + this.supervisor = new Supervisor(new Transport()); + this.target = supervisor.connect(new Spec(rpcPort)); + } + + void invoke(Request request) { + target.invokeSync(request, Duration.ofMinutes(10).getSeconds()); + } + + @Override + public void close() { + target.close(); + supervisor.transport().shutdown().join(); + } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java index cb381253f6a..caeff01f440 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -149,7 +149,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { if (filesOwnedByApplication.contains(requestedFile)) { return; // allowed to access } - throw new AuthorizationException("Peer is not allowed to access file " + requestedFile.value()); + throw new AuthorizationException(String.format("Peer is not allowed to access file %s. Peer is owned by %s", requestedFile.value(), peerOwner.toShortString())); default: throw new AuthorizationException(String.format("'%s' nodes are not allowed to access files", peerIdentity.nodeType())); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java index 22069d0270c..8fdc093122e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java @@ -24,7 +24,7 @@ public class ByteField extends DocsumField { if (value == EMPTY_VALUE) { return NanNumber.NaN; } else { - return Byte.valueOf(value); + return value; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java index a0c9b10c519..2e9d9d8cad9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java @@ -8,27 +8,27 @@ package com.yahoo.prelude.hitfield; */ public final class RawData { - private byte[] content; + private final byte[] content; /** - * Constructor, takes ownership + * Constructor, takes ownership of the given byte array. + * * @param content some bytes, handover */ public RawData(byte[] content) { this.content = content; } - /** - * @return internal byte array containing the actual data received - **/ + /** Returns the internal byte array containing the actual data received */ public byte[] getInternalData() { return content; } /** - * an ascii string; non-ascii data is escaped with hex notation - * NB: not always uniquely reversible - **/ + * An ascii string; non-ascii data is escaped with hex notation. + * NB: not always uniquely reversible. + */ + @Override public String toString() { StringBuilder buf = new StringBuilder(); for (byte b : content) { @@ -46,7 +46,7 @@ public final class RawData { } else { // XXX maybe we should only do this? creates possibly-invalid XML though. buf.append("&"); - buf.append(Integer.toString(i)); + buf.append(i); buf.append(";"); } } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 54dfbfe1a85..af453983f89 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -776,7 +776,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } else { renderInspectorDirect(data); } - } private void renderInspectorDirect(Inspector data) throws IOException { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 3b30e0df9ad..4f54b60187b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -134,7 +134,7 @@ public class FastSearcherTestCase { documentdbConfigWithOneDb); { // No direct.summaries - String query = "?query=sddocname:a&summary=simple"; + String query = "?query=sddocname:a&summary=simple&timeout=20s"; Result result = doSearch(fastSearcher, new Query(query), 0, 10); doFill(fastSearcher, result); ErrorMessage error = result.hits().getError(); @@ -142,7 +142,7 @@ public class FastSearcherTestCase { } { // direct.summaries due to query cache - String query = "?query=sddocname:a&ranking.queryCache&timeout=5000ms"; + String query = "?query=sddocname:a&ranking.queryCache&timeout=20s"; Result result = doSearch(fastSearcher, new Query(query), 0, 10); doFill(fastSearcher, result); ErrorMessage error = result.hits().getError(); @@ -151,7 +151,7 @@ public class FastSearcherTestCase { } { // direct.summaries due to no summary features - String query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler&timeout=5000ms"; + String query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler&timeout=20s"; Result result = doSearch(fastSearcher, new Query(query), 0, 10); doFill(fastSearcher, result); ErrorMessage error = result.hits().getError(); diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index a245d61bafb..1a68f14af06 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -24,6 +24,7 @@ import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.prelude.hitfield.RawData; import com.yahoo.prelude.searcher.JuniperSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -65,6 +66,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -147,7 +149,8 @@ public class JsonRendererTestCase { + " \"scalar2\":2.5," + " \"tensor1\":{\"type\":\"tensor(x[3])\",\"cells\":[{\"address\":{\"x\":\"0\"},\"value\":1.5},{\"address\":{\"x\":\"1\"},\"value\":2.0},{\"address\":{\"x\":\"2\"},\"value\":2.5}]}," + " \"tensor2\":{\"type\":\"tensor()\",\"cells\":[{\"address\":{},\"value\":0.5}]}" - + " }" + + " }," + + " \"data\": \"Data \\\\xc3\\\\xa6 \\\\xc3\\\\xa5\"" + " }," + " \"id\": \"datatypestuff\"," + " \"relevance\": 1.0" @@ -175,6 +178,7 @@ public class JsonRendererTestCase { h.setField("tensor3", Tensor.from("{ {x:a, y:0}: 2.0, {x:a, y:1}: -1 }")); h.setField("object", new Thingie()); h.setField("summaryfeatures", createSummaryFeatures()); + h.setField("data", new RawData("Data æ å".getBytes(StandardCharsets.UTF_8))); r.hits().add(h); r.setTotalHitCount(1L); String summary = render(r); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java deleted file mode 100644 index 87970458855..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef; - -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -/** - * @author mortent - */ -public class AttributeMapping { - - private final String attribute; - private final List<String> chefPath; - - private AttributeMapping(String attribute, List<String> chefPath) { - this.chefPath = chefPath; - this.attribute = attribute; - } - - public static AttributeMapping simpleMapping(String attribute) { - return new AttributeMapping(attribute, Collections.singletonList(attribute)); - } - - public static AttributeMapping deepMapping(String attribute, List<String> chefPath) { - return new AttributeMapping(attribute, chefPath); - } - - public String toString() { - return String.format("\"%s\": [%s]", attribute, - chefPath.stream().map(s -> String.format("\"%s\"", s)) - .collect(Collectors.joining(",")) - ); - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java deleted file mode 100644 index 693947b6f61..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef; - - -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefEnvironment; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefNode; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefResource; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.Client; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.CookBook; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.NodeResult; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; - -import java.net.URL; -import java.util.List; - -public interface Chef { - - ChefResource getApi(); - - ChefNode getNode(String name); - - Client getClient(String name); - - ChefNode deleteNode(String name); - - Client deleteClient(String name); - - NodeResult searchNodeByFQDN(String fqdn); - - NodeResult searchNodes(String query); - - PartialNodeResult partialSearchNodes(String query, List<AttributeMapping> attributeMappings); - - void copyChefEnvironment(String fromEnvironmentName, String toEnvironmentName); - - ChefEnvironment getChefEnvironment(String environmentName); - - CookBook getCookbook(String cookbookName, String cookbookVersion); - - String downloadResource(URL resourceURL); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java deleted file mode 100644 index bd19cfe6ce1..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef; - -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefEnvironment; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefNode; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefResource; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.Client; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.CookBook; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.NodeResult; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; - -import javax.ws.rs.NotFoundException; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author mpolden - */ -public class ChefMock implements Chef { - - private final NodeResult result; - private final PartialNodeResult partialResult; - private final List<String> chefEnvironments; - - public ChefMock() { - result = new NodeResult(); - result.rows = new ArrayList<>(); - partialResult = new PartialNodeResult(); - partialResult.rows = new ArrayList<>(); - chefEnvironments = new ArrayList<>(); - chefEnvironments.add("hosted-verified-prod"); - chefEnvironments.add("hosted-infra-cd"); - } - - @Override - public ChefResource getApi() { - return null; - } - - @Override - public ChefNode getNode(String name) { - return null; - } - - @Override - public Client getClient(String name) { - return null; - } - - @Override - public ChefNode deleteNode(String name) { - return null; - } - - @Override - public Client deleteClient(String name) { - return null; - } - - public ChefMock addSearchResult(ChefNode node) { - result.rows.add(node); - return this; - } - - public ChefMock addPartialResult(List<PartialNode> partialNodes) { - partialResult.rows.addAll(partialNodes); - return this; - } - - @Override - public NodeResult searchNodeByFQDN(String fqdn) { - return result; - } - - @Override - public NodeResult searchNodes(String query) { - return result; - } - - @Override - public PartialNodeResult partialSearchNodes(String query, List<AttributeMapping> returnAttributes) { - PartialNodeResult partialNodeResult = new PartialNodeResult(); - partialNodeResult.rows = new ArrayList<>(); - partialNodeResult.rows.addAll(partialResult.rows); - result.rows.stream() - .map(chefNode -> { - Map<String, String> data = new HashMap<>(); - data.put("fqdn", chefNode.name); - return new PartialNode(data); - }) - .forEach(node -> partialNodeResult.rows.add(node)); - return partialNodeResult; - } - - @Override - public void copyChefEnvironment(String fromEnvironmentName, String toEnvironmentName) { - if(!chefEnvironments.contains(fromEnvironmentName)) { - throw new NotFoundException(String.format("Source chef environment %s does not exist", fromEnvironmentName)); - } - chefEnvironments.add(toEnvironmentName); - } - - @Override - public ChefEnvironment getChefEnvironment(String environmentName) { - return null; - } - - @Override - public CookBook getCookbook(String cookbookName, String cookbookVersion) { - return null; - } - - @Override - public String downloadResource(URL resourceURL) { - return ""; - } -} - diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java deleted file mode 100644 index 5d3d4b87b74..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.api.integration.chef; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java deleted file mode 100644 index 8576949280b..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Map; - -/** - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class ChefEnvironment { - - @JsonProperty("name") - private String name; - - @JsonProperty("default_attributes") - private Map<String, Object> attributes; - @JsonProperty("override_attributes") - private Map<String, Object> overrideAttributes; - @JsonProperty("description") - private String description; - @JsonProperty("cookbook_versions") - private Map<String, String> cookbookVersions; - - // internal - @JsonProperty("json_class") - private final String _jsonClass = "Chef::Environment"; - @JsonProperty("chef_type") - private final String _chefType = "environment"; - - public static Builder builder() { - return new Builder(); - } - - public String getName() { - return name; - } - - public Builder copy() { - return builder() - .name(name) - .attributes(attributes) - .overrideAttributes(overrideAttributes) - .cookbookVersions(cookbookVersions) - .description(description); - } - - public String getDescription() { - return description; - } - - public Map<String, String> getCookbookVersions() { - return cookbookVersions; - } - - public Map<String, Object> getAttributes() { - return attributes; - } - - public Map<String, Object> getOverrideAttributes() { - return overrideAttributes; - } - - public static class Builder { - private String name; - private Map<String, Object> attributes; - private String description; - private Map<String, Object> overrideAttributes; - private Map<String, String> cookbookVersions; - - public Builder name(String name){ - this.name = name; - return this; - } - - public Builder attributes(Map<String, Object> defaultAttributes) { - this.attributes = defaultAttributes; - return this; - } - - public Builder overrideAttributes(Map<String, Object> overrideAttributes) { - this.overrideAttributes = overrideAttributes; - return this; - } - - public Builder cookbookVersions(Map<String, String> cookbookVersions) { - this.cookbookVersions = cookbookVersions; - return this; - } - - public Builder description(String description) { - this.description = description; - return this; - } - - public ChefEnvironment build() { - ChefEnvironment chefEnvironment = new ChefEnvironment(); - chefEnvironment.name = name; - chefEnvironment.description = description; - chefEnvironment.cookbookVersions = cookbookVersions; - chefEnvironment.attributes = attributes; - chefEnvironment.overrideAttributes = overrideAttributes; - - return chefEnvironment; - } - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java deleted file mode 100644 index 08d9a1045e8..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class ChefNode { - - @JsonProperty("name") - public String name; - - @JsonProperty("chef_environment") - public String chefEnvironment; - - @JsonProperty("run_list") - public List<String> runList; - - @JsonProperty("json_class") - public String jsonClass; - - @JsonProperty("chef_type") - public String chefType; - - @JsonProperty("automatic") - public Map<String, Object> automaticAttributes; - - @JsonProperty("normal") - public Map<String, Object> normalAttributes; - - @JsonProperty("default") - public Map<String, Object> defaultAttributes; - - @JsonProperty("override") - public Map<String, Object> overrideAttributes; - - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(ChefNode src) { - return new Builder(src); - } - - public static class Builder { - private String name; - private String chefEnvironment; - private List<String> runList; - private String jsonClass; - private String chefType; - private Map<String, Object> automaticAttributes; - private Map<String, Object> normalAttributes; - private Map<String, Object> defaultAttributes; - private Map<String, Object> overrideAttributes; - - private Builder(){} - - private Builder(ChefNode src){ - this.name = src.name; - this.chefEnvironment = src.chefEnvironment; - this.runList = new ArrayList<>(src.runList); - this.jsonClass = src.jsonClass; - this.chefType = src.chefType; - this.automaticAttributes = new HashMap<>(src.automaticAttributes); - this.normalAttributes = new HashMap<>(src.normalAttributes); - this.defaultAttributes = new HashMap<>(src.defaultAttributes); - this.overrideAttributes = new HashMap<>(src.overrideAttributes); - } - - public Builder name(String name) { - this.name = name; - return this; - } - - public Builder chefEnvironment(String chefEnvironment) { - this.chefEnvironment = chefEnvironment; - return this; - } - - public ChefNode build(){ - ChefNode node = new ChefNode(); - node.name = this.name; - node.chefEnvironment = this.chefEnvironment; - node.runList = this.runList; - node.jsonClass = this.jsonClass; - node.chefType = this.chefType; - node.automaticAttributes = this.automaticAttributes; - node.overrideAttributes = this.overrideAttributes; - node.defaultAttributes = this.defaultAttributes; - node.normalAttributes = this.normalAttributes; - return node; - } - - } - - @Override - public String toString() { - return "Node{" + - "name='" + name + '\'' + - ", chefEnvironment='" + chefEnvironment + '\'' + - ", runList=" + runList + - ", jsonClass='" + jsonClass + '\'' + - ", chefType='" + chefType + '\'' + - ", automaticAttributes=" + automaticAttributes + - ", normalAttributes=" + normalAttributes + - ", defaultAttributes=" + defaultAttributes + - ", overrideAttributes=" + overrideAttributes + - '}'; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java deleted file mode 100644 index 98eeb0770fc..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; -import java.util.List; - -/** - * @author mortent - * @author mpolden - */ - -@Path("/") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public interface ChefResource { - - @Path("/organizations/{organization}/environments/{environment}/nodes") - @Consumes("application/json") - @GET - List<String> getNodes(@PathParam("organization") String organization, @PathParam("environment") String environment); - - @GET - @Path("/organizations/{organization}/nodes/{nodename}") - ChefNode getNode(@PathParam("organization") String organization, @PathParam("nodename") String nodename); - - @PUT - @Path("/organizations/{organization}/nodes/{nodename}") - ChefNode updateNode(@PathParam("organization") String organization, @PathParam("nodename") String nodeName, String node); - - @DELETE - @Path("/organizations/{organization}/nodes/{nodename}") - ChefNode deleteNode(@PathParam("organization") String organization, @PathParam("nodename") String nodeName); - - @GET - @Path("/organizations/{organization}/clients/{name}") - Client getClient(@PathParam("organization") String organization, @PathParam("name") String name); - - @DELETE - @Path("/organizations/{organization}/clients/{name}") - Client deleteClient(@PathParam("organization") String organization, @PathParam("name") String name); - - @GET - @Path("/organizations/{organization}/environments/{environment}") - ChefEnvironment getEnvironment(@PathParam("organization") String organization, @PathParam("environment") String environment); - - @PUT - @Path("/organizations/{organization}/environments/{name}") - String updateEnvironment(@PathParam("organization") String organization, @PathParam("name") String chefEnvironmentName, String contentAsString); - - @POST - @Path("/organizations/{organization}/environments") - String createEnvironment(@PathParam("organization") String organization, String contentAsString); - - @GET - @Path("/organizations/{organization}/search/node") - NodeResult searchNode(@PathParam("organization") String organization, @QueryParam("q") String query); - - @POST - @Path("/organizations/{organization}/search/node") - PartialNodeResult partialSearchNode(@PathParam("organization") String organization, @QueryParam("q") String query, @QueryParam("rows") int rows, String keys); - - @GET - @Path("/organizations/{organization}/cookbooks/{name}/{version}") - CookBook getCookBook(@PathParam("organization") String organization, @PathParam("name") String name, @PathParam("version") String version); -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java deleted file mode 100644 index 0ea9b0e9997..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * @author mpolden - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class Client { - - @JsonProperty("name") - public String name; - @JsonProperty("validator") - public boolean validator; - - @Override - public String toString() { - return "Client{" + - "name='" + name + '\'' + - ", validator=" + validator + - '}'; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java deleted file mode 100644 index ab49ac9ff60..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; - -/** - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class CookBook { - public final String name; - public final List<Attributes> attributes; - - public CookBook(@JsonProperty("name") String name, @JsonProperty("attributes") List<Attributes> attributes) { - this.name = name; - this.attributes = attributes; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Attributes { - public final String name; - public final String url; - - public Attributes(@JsonProperty("name") String name, @JsonProperty("url") String url) { - this.name = name; - this.url = url; - } - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java deleted file mode 100644 index e3ab431473f..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; - -/** - * @author mpolden - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class NodeResult { - @JsonProperty("total") - public int total; - @JsonProperty("start") - public int start; - @JsonProperty("rows") - public List<ChefNode> rows; -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java deleted file mode 100644 index f4aa90021b1..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Map; -import java.util.Optional; - -/** - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class PartialNode { - - @JsonProperty("data") - private final Map<String, String> data; - - @JsonCreator - public PartialNode(@JsonProperty("data") Map<String, String> data) { - this.data = data; - } - - public Optional<String> getValue(String key) { - return Optional.ofNullable(data.get(key)); - } - - public String getFqdn() { - return getValue("fqdn").orElse(""); - } - - public String getName() { - return getValue("name").orElse(""); - } - - public Double getOhaiTime() { - return Double.parseDouble(getValue("ohai_time").orElse("0.0")); - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java deleted file mode 100644 index 9925237a193..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; - -/** - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class PartialNodeResult { - @JsonProperty("total") - public int total; - @JsonProperty("start") - public int start; - @JsonProperty("rows") - public List<PartialNode> rows; -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java deleted file mode 100644 index 7d06571507e..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index ba00203ec34..9eae2965c45 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -30,7 +30,7 @@ public interface ConfigServer { } PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationNames, - List<ContainerEndpoint> containerEndpoints, ApplicationCertificate applicationCertificate, byte[] content); + Set<ContainerEndpoint> containerEndpoints, ApplicationCertificate applicationCertificate, byte[] content); void restart(DeploymentId deployment, Optional<Hostname> hostname); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 6745b75a9ea..1ec75e0c998 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -141,9 +141,7 @@ enum PathGroup { Matcher.application, Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/jobreport", - "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/jobreport", - "/application/v4/tenant/{tenant}/application/{application}/promote", - "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote"), + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/jobreport"), /** Paths which contain (not very strictly) classified information about customers. */ classifiedTenantInfo(Optional.of("/api"), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 9ca73d27120..d8b56502fc3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -218,12 +219,18 @@ public class Application { return rotations; } + /** Returns the default global endpoints for this in given system - for a given endpoint ID */ + public EndpointList endpointsIn(SystemName system, EndpointId endpointId) { + if (rotations.isEmpty()) return EndpointList.EMPTY; + return EndpointList.create(id, endpointId, system); + } + /** Returns the default global endpoints for this in given system */ public EndpointList endpointsIn(SystemName system) { - // TODO: Do we need to change something here? .defaultGlobalId seems like it is - // TODO: making some assumptions on naming. if (rotations.isEmpty()) return EndpointList.EMPTY; - return EndpointList.defaultGlobal(id, system); + final var endpointStream = rotations.stream() + .flatMap(rotation -> EndpointList.create(id, rotation.endpointId(), system).asList().stream()); + return EndpointList.of(endpointStream); } public Optional<String> pemDeployKey() { return pemDeployKey; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 197dda8c409..d015e65a5e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller; import com.google.common.collect.ImmutableList; +import com.yahoo.collections.ArraySet; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; @@ -9,6 +10,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -30,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.certificates.Applicatio import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException; @@ -87,6 +90,7 @@ import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -97,6 +101,7 @@ import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; @@ -126,6 +131,7 @@ public class ApplicationController { private final RoutingPolicies routingPolicies; private final Clock clock; private final BooleanFlag redirectLegacyDnsFlag; + private final BooleanFlag useMultipleEndpoints; private final DeploymentTrigger deploymentTrigger; private final BooleanFlag provisionApplicationCertificate; private final ApplicationCertificateProvider applicationCertificateProvider; @@ -143,6 +149,7 @@ public class ApplicationController { this.routingPolicies = new RoutingPolicies(controller); this.clock = clock; this.redirectLegacyDnsFlag = Flags.REDIRECT_LEGACY_DNS_NAMES.bindTo(controller.flagSource()); + this.useMultipleEndpoints = Flags.MULTIPLE_GLOBAL_ENDPOINTS.bindTo(controller.flagSource()); this.artifactRepository = artifactRepository; this.applicationStore = applicationStore; @@ -293,7 +300,8 @@ public class ApplicationController { Version platformVersion; ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; - Set<String> rotationNames = new HashSet<>(); + Set<String> legacyRotations = new LinkedHashSet<>(); + Set<ContainerEndpoint> endpoints = new LinkedHashSet<>(); ApplicationCertificate applicationCertificate; try (Lock lock = lock(applicationId)) { @@ -332,13 +340,34 @@ public class ApplicationController { // TODO: Remove this when all packages are validated upon submission, as in ApplicationApiHandler.submit(...). verifyApplicationIdentityConfiguration(applicationId.tenant(), applicationPackage, deployingIdentity); + // Assign global rotation - application = withRotation(application, zone); - Application app = application.get(); - // Include global DNS names - app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).forEach(rotationNames::add); - // Include rotation ID to ensure that deployment can respond to health checks with rotation ID as Host header - app.rotations().stream().map(RotationId::asString).forEach(rotationNames::add); + if (useMultipleEndpoints.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()).value()) { + application = withRotation(application, zone); + + // Include global DNS names + Application app = application.get(); + app.assignedRotations().stream() + .filter(assignedRotation -> assignedRotation.regions().contains(zone.region())) + .map(assignedRotation -> { + return new ContainerEndpoint( + assignedRotation.clusterId().value(), + Stream.concat( + app.endpointsIn(controller.system(), assignedRotation.endpointId()).legacy(false).asList().stream().map(Endpoint::dnsName), + app.rotations().stream().map(RotationId::asString) + ).collect(Collectors.toList()) + ); + }) + .forEach(endpoints::add); + } else { + application = withRotationLegacy(application, zone); + + // Add both the names we have in DNS for each endpoint as well as name of the rotation so healthchecks works + Application app = application.get(); + app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).forEach(legacyRotations::add); + app.rotations().stream().map(RotationId::asString).forEach(legacyRotations::add); + } + // Get application certificate (provisions a new certificate if missing) application = withApplicationCertificate(application); @@ -354,7 +383,7 @@ public class ApplicationController { // Carry out deployment without holding the application lock. options = withVersion(platformVersion, options); - ActivateResult result = deploy(applicationId, applicationPackage, zone, options, rotationNames, applicationCertificate); + ActivateResult result = deploy(applicationId, applicationPackage, zone, options, legacyRotations, endpoints, applicationCertificate); lockOrThrow(applicationId, application -> store(application.withNewDeployment(zone, applicationVersion, platformVersion, clock.instant(), @@ -421,7 +450,7 @@ public class ApplicationController { artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); DeployOptions options = withVersion(version, DeployOptions.none()); - return deploy(application.id(), applicationPackage, zone, options, Set.of(), /* No application cert */ null); + return deploy(application.id(), applicationPackage, zone, options, Set.of(), Set.of(), /* No application cert */ null); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -429,16 +458,16 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) { - return deploy(tester.id(), applicationPackage, zone, options, Set.of(), /* No application cert for tester*/ null); + return deploy(tester.id(), applicationPackage, zone, options, Set.of(), Set.of(), /* No application cert for tester*/ null); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions deployOptions, - Set<String> rotationNames, ApplicationCertificate applicationCertificate) { + Set<String> legacyRotations, Set<ContainerEndpoint> endpoints, ApplicationCertificate applicationCertificate) { DeploymentId deploymentId = new DeploymentId(application, zone); try { ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(deploymentId, deployOptions, rotationNames, List.of(), applicationCertificate, applicationPackage.zippedContent()); + configServer.deploy(deploymentId, deployOptions, legacyRotations, endpoints, applicationCertificate, applicationPackage.zippedContent()); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { @@ -449,19 +478,20 @@ public class ApplicationController { } /** Makes sure the application has a global rotation, if eligible. */ - private LockedApplication withRotation(LockedApplication application, ZoneId zone) { + private LockedApplication withRotationLegacy(LockedApplication application, ZoneId zone) { if (zone.environment() == Environment.prod && application.get().deploymentSpec().globalServiceId().isPresent()) { try (RotationLock rotationLock = rotationRepository.lock()) { Rotation rotation = rotationRepository.getOrAssignRotation(application.get(), rotationLock); - application = application.with(List.of(new AssignedRotation(new ClusterSpec.Id(application.get().deploymentSpec().globalServiceId().get()), EndpointId.default_(), rotation.id()))); + application = application.with(createDefaultGlobalIdRotation(application.get(), rotation)); store(application); // store assigned rotation even if deployment fails boolean redirectLegacyDns = redirectLegacyDnsFlag.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()) - .value(); + .value(); EndpointList globalEndpoints = application.get() - .endpointsIn(controller.system()) - .scope(Endpoint.Scope.global); + .endpointsIn(controller.system()) + .scope(Endpoint.Scope.global); + globalEndpoints.main().ifPresent(mainEndpoint -> { registerCname(mainEndpoint.dnsName(), rotation.name()); if (redirectLegacyDns) { @@ -475,6 +505,54 @@ public class ApplicationController { return application; } + private List<AssignedRotation> createDefaultGlobalIdRotation(Application application, Rotation rotation) { + // This is guaranteed by .withRotationLegacy, but add this to make inspections accept the use of .get() below + assert application.deploymentSpec().globalServiceId().isPresent(); + + final Set<RegionName> regions = application.deploymentSpec().zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); + + final var assignment = new AssignedRotation( + ClusterSpec.Id.from(application.deploymentSpec().globalServiceId().get()), + EndpointId.default_(), + rotation.id(), + regions + ); + + return List.of(assignment); + } + + /** Makes sure the application has a global rotation, if eligible. */ + private LockedApplication withRotation(LockedApplication application, ZoneId zone) { + if (zone.environment() == Environment.prod) { + try (RotationLock rotationLock = rotationRepository.lock()) { + final var rotations = rotationRepository.getOrAssignRotations(application.get(), rotationLock); + application = application.with(rotations); + store(application); // store assigned rotation even if deployment fails + registerAssignedRotationCnames(application.get()); + } + } + return application; + } + + private void registerAssignedRotationCnames(Application application) { + application.assignedRotations().forEach(assignedRotation -> { + final var endpoints = application + .endpointsIn(controller.system(), assignedRotation.endpointId()) + .scope(Endpoint.Scope.global); + + final var maybeRotation = rotationRepository.getRotation(assignedRotation.rotationId()); + + maybeRotation.ifPresent(rotation -> { + endpoints.main().ifPresent(mainEndpoint -> { + registerCname(mainEndpoint.dnsName(), rotation.name()); + }); + }); + }); + } + private LockedApplication withApplicationCertificate(LockedApplication application) { ApplicationId applicationId = application.get().id(); @@ -631,8 +709,14 @@ public class ApplicationController { applicationStore.removeAll(id); applicationStore.removeAll(TesterId.of(id)); - EndpointList endpoints = application.get().endpointsIn(controller.system()); - endpoints.asList().stream().map(Endpoint::dnsName).forEach(name -> controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal)); + application.get().assignedRotations().forEach(assignedRotation -> { + final var endpoints = application.get().endpointsIn(controller.system(), assignedRotation.endpointId()); + endpoints.asList().stream() + .map(Endpoint::dnsName) + .forEach(name -> { + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal); + }); + }); log.info("Deleted " + application); })); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index ed81d08c533..08c95d1ecab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; @@ -79,7 +78,6 @@ public class Controller extends AbstractComponent { private final ZoneRegistry zoneRegistry; private final ConfigServer configServer; private final MetricsService metricsService; - private final Chef chef; private final Mailer mailer; private final AuditLogger auditLogger; private final FlagSource flagSource; @@ -95,13 +93,13 @@ public class Controller extends AbstractComponent { @Inject public Controller(CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, ZoneRegistry zoneRegistry, ConfigServer configServer, MetricsService metricsService, - RoutingGenerator routingGenerator, Chef chef, + RoutingGenerator routingGenerator, AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, BuildService buildService, RunDataStore runDataStore, Mailer mailer, FlagSource flagSource, MavenRepository mavenRepository, ApplicationCertificateProvider applicationCertificateProvider) { this(curator, rotationsConfig, gitHub, zoneRegistry, - configServer, metricsService, routingGenerator, chef, + configServer, metricsService, routingGenerator, Clock.systemUTC(), accessControl, artifactRepository, applicationStore, testerCloud, buildService, runDataStore, com.yahoo.net.HostName::getLocalhost, mailer, flagSource, mavenRepository, applicationCertificateProvider); @@ -110,7 +108,7 @@ public class Controller extends AbstractComponent { public Controller(CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, ZoneRegistry zoneRegistry, ConfigServer configServer, MetricsService metricsService, - RoutingGenerator routingGenerator, Chef chef, Clock clock, + RoutingGenerator routingGenerator, Clock clock, AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, BuildService buildService, RunDataStore runDataStore, Supplier<String> hostnameSupplier, @@ -122,7 +120,6 @@ public class Controller extends AbstractComponent { this.zoneRegistry = Objects.requireNonNull(zoneRegistry, "ZoneRegistry cannot be null"); this.configServer = Objects.requireNonNull(configServer, "ConfigServer cannot be null"); this.metricsService = Objects.requireNonNull(metricsService, "MetricsService cannot be null"); - this.chef = Objects.requireNonNull(chef, "Chef cannot be null"); this.clock = Objects.requireNonNull(clock, "Clock cannot be null"); this.mailer = Objects.requireNonNull(mailer, "Mailer cannot be null"); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); @@ -294,10 +291,6 @@ public class Controller extends AbstractComponent { return zoneRegistry.system(); } - public Chef chefClient() { - return chef; - } - public CuratorDb curator() { return curator; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java index e1ed278a79e..ec13066d069 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -1,12 +1,16 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.rotation.RotationId; +import java.util.Collection; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; /** - * Contains the tuple of [clusterId, endpointId, rotationId], to keep track + * Contains the tuple of [clusterId, endpointId, rotationId, regions[]], to keep track * of which services have assigned which rotations under which name. * * @author ogronnesby @@ -15,16 +19,19 @@ public class AssignedRotation { private final ClusterSpec.Id clusterId; private final EndpointId endpointId; private final RotationId rotationId; + private final Set<RegionName> regions; - public AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, RotationId rotationId) { + public AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, RotationId rotationId, Set<RegionName> regions) { this.clusterId = requireNonEmpty(clusterId, clusterId.value(), "clusterId"); this.endpointId = Objects.requireNonNull(endpointId); this.rotationId = Objects.requireNonNull(rotationId); + this.regions = Set.copyOf(Objects.requireNonNull(regions)); } public ClusterSpec.Id clusterId() { return clusterId; } public EndpointId endpointId() { return endpointId; } public RotationId rotationId() { return rotationId; } + public Set<RegionName> regions() { return regions; } @Override public String toString() { @@ -32,6 +39,7 @@ public class AssignedRotation { "clusterId=" + clusterId + ", endpointId='" + endpointId + '\'' + ", rotationId=" + rotationId + + ", regions=" + regions + '}'; } @@ -42,12 +50,13 @@ public class AssignedRotation { AssignedRotation that = (AssignedRotation) o; return clusterId.equals(that.clusterId) && endpointId.equals(that.endpointId) && - rotationId.equals(that.rotationId); + rotationId.equals(that.rotationId) && + regions.equals(that.regions); } @Override public int hashCode() { - return Objects.hash(clusterId, endpointId, rotationId); + return Objects.hash(clusterId, endpointId, rotationId, regions); } private static <T> T requireNonEmpty(T object, String value, String field) { @@ -58,4 +67,14 @@ public class AssignedRotation { } return object; } + + /** Convenience method intended for tests */ + public static AssignedRotation fromStrings(String clusterId, String endpointId, String rotationId, Collection<String> regions) { + return new AssignedRotation( + new ClusterSpec.Id(clusterId), + new EndpointId(endpointId), + new RotationId(rotationId), + regions.stream().map(RegionName::from).collect(Collectors.toSet()) + ); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 5026ca75a83..5dccd5c8120 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -217,6 +217,7 @@ public class Endpoint { private ZoneId zone; private ClusterSpec.Id cluster; private RotationName rotation; + private EndpointId endpointId; private Port port; private boolean legacy = false; private boolean directRouting = false; @@ -227,8 +228,8 @@ public class Endpoint { /** Sets the cluster and zone target of this */ public EndpointBuilder target(ClusterSpec.Id cluster, ZoneId zone) { - if (rotation != null) { - throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + if (rotation != null || endpointId != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); } this.cluster = cluster; this.zone = zone; @@ -237,13 +238,22 @@ public class Endpoint { /** Sets the rotation target of this */ public EndpointBuilder target(RotationName rotation) { - if (cluster != null && zone != null) { - throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + if ((cluster != null && zone != null) || endpointId != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); } this.rotation = rotation; return this; } + /** Sets the endpoint ID as defines in deployments.xml */ + public EndpointBuilder named(EndpointId endpointId) { + if (rotation != null || cluster != null || zone != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); + } + this.endpointId = endpointId; + return this; + } + /** Sets the port of this */ public EndpointBuilder on(Port port) { this.port = port; @@ -269,6 +279,8 @@ public class Endpoint { name = cluster.value(); } else if (rotation != null) { name = rotation.value(); + } else if (endpointId != null) { + name = endpointId.id(); } else { throw new IllegalArgumentException("Must set either cluster or rotation target"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index 0c04a1f099c..d9aea783880 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -25,13 +25,6 @@ public class EndpointList { private final List<Endpoint> endpoints; private EndpointList(List<Endpoint> endpoints) { - long mainEndpoints = endpoints.stream() - .filter(endpoint -> endpoint.scope() == Endpoint.Scope.global) - .filter(Predicate.not(Endpoint::directRouting)) - .filter(Predicate.not(Endpoint::legacy)).count(); - if (mainEndpoints > 1) { - throw new IllegalArgumentException("Can have only 1 non-legacy global endpoint, got " + endpoints); - } if (endpoints.stream().distinct().count() != endpoints.size()) { throw new IllegalArgumentException("Expected all endpoints to be distinct, got " + endpoints); } @@ -67,16 +60,14 @@ public class EndpointList { } /** Returns the default global endpoints in given system. Default endpoints are served by a pre-provisioned routing layer */ - public static EndpointList defaultGlobal(ApplicationId application, SystemName system) { - // Rotation name is always default in the routing layer - RotationName rotation = RotationName.from("default"); + public static EndpointList create(ApplicationId application, EndpointId endpointId, SystemName system) { switch (system) { case cd: case main: return new EndpointList(List.of( - Endpoint.of(application).target(rotation).on(Port.plain(4080)).legacy().in(system), - Endpoint.of(application).target(rotation).on(Port.tls(4443)).legacy().in(system), - Endpoint.of(application).target(rotation).on(Port.tls(4443)).in(system) + Endpoint.of(application).named(endpointId).on(Port.plain(4080)).legacy().in(system), + Endpoint.of(application).named(endpointId).on(Port.tls(4443)).legacy().in(system), + Endpoint.of(application).named(endpointId).on(Port.tls(4443)).in(system) )); } return EMPTY; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index e840deb062c..f34c24c497a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -5,15 +5,14 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; -import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; -import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; +import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; +import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; @@ -59,7 +58,7 @@ public class ControllerMaintenance extends AbstractComponent { @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, - JobControl jobControl, Metric metric, Chef chefClient, + JobControl jobControl, Metric metric, DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues, NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient, ContactRetriever contactRetriever, @@ -71,7 +70,7 @@ public class ControllerMaintenance extends AbstractComponent { this.jobControl = jobControl; deploymentExpirer = new DeploymentExpirer(controller, maintenanceInterval, jobControl); deploymentIssueReporter = new DeploymentIssueReporter(controller, deploymentIssues, maintenanceInterval, jobControl); - metricsReporter = new MetricsReporter(controller, metric, chefClient, jobControl, controller.system()); + metricsReporter = new MetricsReporter(controller, metric, jobControl); outstandingChangeDeployer = new OutstandingChangeDeployer(controller, Duration.ofMinutes(1), jobControl); versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(1), jobControl); upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 5e3f21c6b98..c7b76696d84 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -3,14 +3,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.SystemName; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.chef.AttributeMapping; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -24,10 +19,8 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; /** @@ -36,7 +29,6 @@ import java.util.stream.Collectors; */ public class MetricsReporter extends Maintainer { - public static final String CONVERGENCE_METRIC = "seconds.since.last.chef.convergence"; public static final String DEPLOYMENT_FAIL_METRIC = "deployment.failurePercentage"; public static final String DEPLOYMENT_AVERAGE_DURATION = "deployment.averageDuration"; public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades"; @@ -46,27 +38,16 @@ public class MetricsReporter extends Maintainer { public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; private final Metric metric; - private final Chef chefClient; private final Clock clock; - private final SystemName system; - public MetricsReporter(Controller controller, Metric metric, Chef chefClient, JobControl jobControl, - SystemName system) { - this(controller, metric, chefClient, Clock.systemUTC(), jobControl, system); - } - - public MetricsReporter(Controller controller, Metric metric, Chef chefClient, Clock clock, - JobControl jobControl, SystemName system) { + public MetricsReporter(Controller controller, Metric metric, JobControl jobControl) { super(controller, Duration.ofMinutes(1), jobControl); // use fixed rate for metrics this.metric = metric; - this.chefClient = chefClient; - this.clock = clock; - this.system = system; + this.clock = controller.clock(); } @Override public void maintain() { - reportChefMetrics(); reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); @@ -79,49 +60,6 @@ public class MetricsReporter extends Maintainer { } } - private void reportChefMetrics() { - String query = "chef_environment:hosted*"; - if (system == SystemName.cd) { - query += " AND hosted_system:" + system; - } - PartialNodeResult nodeResult = chefClient.partialSearchNodes(query, - List.of( - AttributeMapping.simpleMapping("fqdn"), - AttributeMapping.simpleMapping("ohai_time"), - AttributeMapping.deepMapping("tenant", List.of("hosted", "owner", "tenant")), - AttributeMapping.deepMapping("application", List.of("hosted", "owner", "application")), - AttributeMapping.deepMapping("instance", List.of("hosted", "owner", "instance")), - AttributeMapping.deepMapping("environment", List.of("hosted", "environment")), - AttributeMapping.deepMapping("region", List.of("hosted", "region")), - AttributeMapping.deepMapping("system", List.of("hosted", "system")) - )); - - // The above search will return a correct list if the system is CD. However for main, it will - // return all nodes, since system==nil for main - keepNodesWithSystem(nodeResult, system); - - Instant instant = clock.instant(); - for (PartialNode node : nodeResult.rows) { - String hostname = node.getFqdn(); - long secondsSinceConverge = Duration.between(Instant.ofEpochSecond(node.getOhaiTime().longValue()), instant).getSeconds(); - Map<String, String> dimensions = new HashMap<>(); - dimensions.put("host", hostname); - dimensions.put("system", node.getValue("system").orElse("main")); - Optional<String> environment = node.getValue("environment"); - Optional<String> region = node.getValue("region"); - - if (environment.isPresent() && region.isPresent()) { - dimensions.put("zone", String.format("%s.%s", environment.get(), region.get())); - } - - node.getValue("tenant").ifPresent(tenant -> dimensions.put("tenantName", tenant)); - Optional<String> application = node.getValue("application"); - application.ifPresent(app -> dimensions.put("app", String.format("%s.%s", app, node.getValue("instance").orElse("default")))); - Metric.Context context = metric.createContext(dimensions); - metric.set(CONVERGENCE_METRIC, secondsSinceConverge, context); - } - } - private void reportDeploymentMetrics() { ApplicationList applications = ApplicationList.from(controller().applications().asList()) .hasProductionDeployment(); @@ -210,10 +148,6 @@ public class MetricsReporter extends Maintainer { .max(Integer::compareTo) .orElse(0); } - - private static void keepNodesWithSystem(PartialNodeResult nodeResult, SystemName system) { - nodeResult.rows.removeIf(node -> !system.value().equals(node.getValue("system").orElse("main"))); - } private static Map<String, String> dimensions(ApplicationId application) { return ImmutableMap.of( diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 6ecf60e7404..0c045eb7253 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -41,6 +41,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -49,6 +50,7 @@ import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.TreeMap; +import java.util.stream.Collectors; /** * Serializes {@link Application} to/from slime. @@ -551,23 +553,25 @@ public class ApplicationSerializer { } private List<AssignedRotation> assignedRotationsFromSlime(DeploymentSpec deploymentSpec, Inspector root) { - final var assignedRotations = new LinkedHashSet<AssignedRotation>(); + final var assignedRotations = new LinkedHashMap<EndpointId, AssignedRotation>(); // Add the legacy rotation field to the set - this needs to be first // TODO: Remove when we retire the rotations field final var legacyRotation = legacyRotationFromSlime(root.field(deprecatedRotationField)); if (legacyRotation.isPresent() && deploymentSpec.globalServiceId().isPresent()) { final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get()); - assignedRotations.add(new AssignedRotation(clusterId, EndpointId.default_(), legacyRotation.get())); + final var regions = deploymentSpec.zones().stream().flatMap(zone -> zone.region().stream()).collect(Collectors.toSet()); + assignedRotations.putIfAbsent(EndpointId.default_(), new AssignedRotation(clusterId, EndpointId.default_(), legacyRotation.get(), regions)); } // Now add the same entries from "stupid" list of rotations // TODO: Remove when we retire the rotations field final var rotations = rotationListFromSlime(root.field(rotationsField)); for (var rotation : rotations) { + final var regions = deploymentSpec.zones().stream().flatMap(zone -> zone.region().stream()).collect(Collectors.toSet()); if (deploymentSpec.globalServiceId().isPresent()) { final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get()); - assignedRotations.add(new AssignedRotation(clusterId, EndpointId.default_(), rotation)); + assignedRotations.putIfAbsent(EndpointId.default_(), new AssignedRotation(clusterId, EndpointId.default_(), rotation, regions)); } } @@ -576,10 +580,14 @@ public class ApplicationSerializer { final var clusterId = new ClusterSpec.Id(inspector.field(assignedRotationClusterField).asString()); final var endpointId = EndpointId.of(inspector.field(assignedRotationEndpointField).asString()); final var rotationId = new RotationId(inspector.field(assignedRotationRotationField).asString()); - assignedRotations.add(new AssignedRotation(clusterId, endpointId, rotationId)); + final var regions = deploymentSpec.endpoints().stream() + .filter(endpoint -> endpoint.endpointId().equals(endpointId.id())) + .flatMap(endpoint -> endpoint.regions().stream()) + .collect(Collectors.toSet()); + assignedRotations.putIfAbsent(endpointId, new AssignedRotation(clusterId, endpointId, rotationId, regions)); }); - return List.copyOf(assignedRotations); + return List.copyOf(assignedRotations.values()); } private List<RotationId> rotationListFromSlime(Inspector field) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 9f091061596..9c320df2f6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -7,15 +7,13 @@ import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -31,9 +29,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.ActivateResult; -import com.yahoo.vespa.hosted.controller.api.application.v4.ApplicationResource; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; -import com.yahoo.vespa.hosted.controller.api.application.v4.TenantResource; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; @@ -50,7 +46,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterCost; @@ -71,7 +66,6 @@ import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; -import com.yahoo.vespa.hosted.controller.restapi.StringResponse; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -220,7 +214,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handlePOST(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), "default", request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", request); @@ -238,11 +231,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/promote")) return promoteApplicationDeployment(path.get("tenant"), path.get("application"), path.get("environment"), path.get("region"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote")) return promoteApplicationDeployment(path.get("tenant"), path.get("application"), path.get("environment"), path.get("region"), path.get("instance"), request); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -273,7 +264,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handleOPTIONS() { // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother // spelling out the methods supported at each path, which we should - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,PUT,POST,PATCH,DELETE,OPTIONS"); return response; } @@ -951,12 +942,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Optional<Hostname> hostname = Optional.ofNullable(request.getProperty("hostname")).map(Hostname::new); controller.applications().restart(deploymentId, hostname); - // TODO: Change to return JSON - return new StringResponse("Requested restart of " + path(TenantResource.API_PATH, tenantName, - ApplicationResource.API_PATH, applicationName, - EnvironmentResource.API_PATH, environment, - "region", region, - "instance", instanceName)); + return new MessageResponse("Requested restart of " + deploymentId); } private HttpResponse jobDeploy(ApplicationId id, JobType type, HttpRequest request) { @@ -1103,53 +1089,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ? Optional.empty() : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); controller.applications().deleteApplication(id, credentials); - return new EmptyJsonResponse(); // TODO: Replicates current behavior but should return a message response instead + return new MessageResponse("Deleted application " + id); } private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName)); // Attempt to deactivate application even if the deployment is not known by the controller - controller.applications().deactivate(application.id(), ZoneId.from(environment, region)); - - // TODO: Change to return JSON - return new StringResponse("Deactivated " + path(TenantResource.API_PATH, tenantName, - ApplicationResource.API_PATH, applicationName, - "instance", instanceName, - EnvironmentResource.API_PATH, environment, - "region", region)); - } + DeploymentId deploymentId = new DeploymentId(application.id(), ZoneId.from(environment, region)); + controller.applications().deactivate(deploymentId.applicationId(), deploymentId.zoneId()); - /** - * Promote application Chef environments. To be used by component jobs only - */ - private HttpResponse promoteApplication(String tenantName, String applicationName, HttpRequest request) { - try{ - ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system()); - String sourceEnvironment = chefEnvironment.systemChefEnvironment(); - String targetEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName)); - controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment); - return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment)); - } catch (Exception e) { - log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s)", tenantName, applicationName), e); - return ErrorResponse.internalServerError("Unable to promote Chef environments for application"); - } - } - - /** - * Promote application Chef environments for jobs that deploy applications - */ - private HttpResponse promoteApplicationDeployment(String tenantName, String applicationName, String environmentName, String regionName, String instanceName, HttpRequest request) { - try { - ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system()); - String sourceEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName)); - String targetEnvironment = chefEnvironment.applicationTargetEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName), Environment.from(environmentName), RegionName.from(regionName)); - controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment); - return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment)); - } catch (Exception e) { - log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s %s.%s)", tenantName, applicationName, environmentName, regionName), e); - return ErrorResponse.internalServerError("Unable to promote Chef environments for application"); - } + return new MessageResponse("Deactivated " + deploymentId); } private HttpResponse notifyJobCompletion(String tenant, String application, HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java deleted file mode 100644 index 7c32e48e218..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; - -/** - * Represents Chef environments for applications/deployments. Used for promotion of Chef environments - * - * @author mortent - */ -public class ApplicationChefEnvironment { - - private final String systemChefEnvironment; - private final String systemSuffix; - - public ApplicationChefEnvironment(SystemName system) { - if (system == SystemName.main) { - systemChefEnvironment = "hosted-verified-prod"; - systemSuffix = ""; - } else { - systemChefEnvironment = "hosted-infra-cd"; - systemSuffix = "-cd"; - } - } - - public String systemChefEnvironment() { - return systemChefEnvironment; - } - - public String applicationSourceEnvironment(TenantName tenantName, ApplicationName applicationName) { - // placeholder and component already used in legacy chef promotion - return String.format("hosted-instance%s_%s_%s_placeholder_component_default", systemSuffix, tenantName, applicationName); - } - - public String applicationTargetEnvironment(TenantName tenantName, ApplicationName applicationName, Environment environment, RegionName regionName) { - return String.format("hosted-instance%s_%s_%s_%s_%s_default", systemSuffix, tenantName, applicationName, regionName, environment); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java index be3222cc1a8..e343615f066 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java @@ -8,16 +8,13 @@ import java.io.OutputStream; /** * @author bratseth */ -public class EmptyJsonResponse extends HttpResponse { +public class EmptyResponse extends HttpResponse { - public EmptyJsonResponse() { + public EmptyResponse() { super(200); } @Override public void render(OutputStream stream) {} - @Override - public String getContentType() { return "application/json"; } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index cd7c0d6236d..b34ea79c670 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -6,26 +6,26 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.NotExistsException; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.deployment.JobController; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; -import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; @@ -94,14 +94,14 @@ class JobControllerApiHandlerHelper { Slime slime = new Slime(); Cursor responseObject = slime.setObject(); + Cursor lastVersionsObject = responseObject.setObject("lastVersions"); if (application.deploymentJobs().statusOf(component).flatMap(JobStatus::lastSuccess).isPresent()) { - Cursor lastVersionsObject = responseObject.setObject("lastVersions"); lastPlatformToSlime(lastVersionsObject.setObject("platform"), controller, application, change, steps); lastApplicationToSlime(lastVersionsObject.setObject("application"), application, change, steps, controller); } + Cursor deployingObject = responseObject.setObject("deploying"); if ( ! change.isEmpty()) { - Cursor deployingObject = responseObject.setObject("deploying"); change.platform().ifPresent(version -> deployingObject.setString("platform", version.toString())); change.application().ifPresent(version -> applicationVersionToSlime(deployingObject.setObject("application"), version)); } @@ -141,7 +141,7 @@ class JobControllerApiHandlerHelper { && type.environment().isManuallyDeployed() && application.deployments().containsKey(type.zone(controller.system()))) controller.jobController().last(application.id(), type) - .ifPresent(last -> runToSlime(devJobsObject.setObject(type.jobName()), + .ifPresent(last -> runToSlime(devJobsObject.setObject(type.jobName()).setArray("runs").addObject(), last, baseUriForJobs.resolve(baseUriForJobs.getPath() + "/" + type.jobName()).normalize())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 978b7e4397d..44b67a186b8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -8,18 +8,18 @@ import com.yahoo.config.provision.HostName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; import com.yahoo.vespa.hosted.controller.restapi.Uri; -import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; -import com.yahoo.restapi.Path; +import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.util.Optional; @@ -71,7 +71,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { private HttpResponse handleOPTIONS() { // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother // spelling out the methods supported at each path, which we should - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,OPTIONS"); return response; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 5ef997b6d55..7a76f13392d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -13,20 +13,19 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; -import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; -import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; +import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; import com.yahoo.yolean.Exceptions; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -99,7 +98,7 @@ public class UserApiHandler extends LoggingRequestHandler { } private HttpResponse handleOPTIONS() { - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,PUT,POST,PATCH,DELETE,OPTIONS"); return response; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java index d2b16721503..f2bc50ec445 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -1,18 +1,30 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; +import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.Endpoint; +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -50,6 +62,11 @@ public class RotationRepository { return application.rotations().stream().map(allRotations::get).findFirst(); } + /** Get rotation for the given rotationId */ + public Optional<Rotation> getRotation(RotationId rotationId) { + return Optional.of(allRotations.get(rotationId)); + } + /** * Returns a rotation for the given application * @@ -76,6 +93,116 @@ public class RotationRepository { } /** + * Returns rotation assignments for all endpoints in application. + * + * If rotations are already assigned, these will be returned. + * If rotations are not assigned, a new assignment will be created taking new rotations from the repository. + * This method supports both global-service-id as well as the new endpoints tag. + * + * @param application The application requesting rotations. + * @param lock Lock which by acquired by the caller + * @return List of rotation assignments - either new or existing. + */ + public List<AssignedRotation> getOrAssignRotations(Application application, RotationLock lock) { + if (application.deploymentSpec().globalServiceId().isPresent() && ! application.deploymentSpec().endpoints().isEmpty()) { + throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); + } + + // Support the older case of setting global-service-id + if (application.deploymentSpec().globalServiceId().isPresent()) { + final var regions = application.deploymentSpec().zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); + + final var rotation = getOrAssignRotation(application, lock); + + return List.of( + new AssignedRotation( + new ClusterSpec.Id(application.deploymentSpec().globalServiceId().get()), + EndpointId.default_(), + rotation.id(), + regions + ) + ); + } + + final Map<EndpointId, AssignedRotation> existingAssignments = existingEndpointAssignments(application); + final Map<EndpointId, AssignedRotation> updatedAssignments = assignRotationsToEndpoints(application, existingAssignments, lock); + + existingAssignments.putAll(updatedAssignments); + + return List.copyOf(existingAssignments.values()); + } + + private Map<EndpointId, AssignedRotation> assignRotationsToEndpoints(Application application, Map<EndpointId, AssignedRotation> existingAssignments, RotationLock lock) { + final var availableRotations = new ArrayList<>(availableRotations(lock).values()); + + final var neededRotations = application.deploymentSpec().endpoints().stream() + .filter(Predicate.not(endpoint -> existingAssignments.containsKey(EndpointId.of(endpoint.endpointId())))) + .collect(Collectors.toSet()); + + if (neededRotations.size() > availableRotations.size()) { + throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation: need " + neededRotations.size() + ", have " + availableRotations.size()); + } + + return neededRotations.stream() + .map(endpoint -> { + return new AssignedRotation( + new ClusterSpec.Id(endpoint.containerId()), + EndpointId.of(endpoint.endpointId()), + availableRotations.remove(0).id(), + endpoint.regions() + ); + }) + .collect( + Collectors.toMap( + AssignedRotation::endpointId, + Function.identity(), + (a, b) -> { throw new IllegalStateException("Duplicate entries:" + a + ", " + b); }, + LinkedHashMap::new + ) + ); + } + + private Map<EndpointId, AssignedRotation> existingEndpointAssignments(Application application) { + // + // Get the regions that has been configured for an endpoint. Empty set if the endpoint + // is no longer mentioned in the configuration file. + // + final Function<EndpointId, Set<RegionName>> configuredRegionsForEndpoint = endpointId -> { + return application.deploymentSpec().endpoints().stream() + .filter(endpoint -> endpointId.id().equals(endpoint.endpointId())) + .map(Endpoint::regions) + .findFirst() + .orElse(Set.of()); + }; + + // + // Build a new AssignedRotation instance where we update set of regions from the configuration instead + // of using the one already mentioned in the assignment. This allows us to overwrite the set of regions + // when + final Function<AssignedRotation, AssignedRotation> assignedRotationWithConfiguredRegions = assignedRotation -> { + return new AssignedRotation( + assignedRotation.clusterId(), + assignedRotation.endpointId(), + assignedRotation.rotationId(), + configuredRegionsForEndpoint.apply(assignedRotation.endpointId()) + ); + }; + + return application.assignedRotations().stream() + .collect( + Collectors.toMap( + AssignedRotation::endpointId, + assignedRotationWithConfiguredRegions, + (a, b) -> { throw new IllegalStateException("Duplicate entries: " + a + ", " + b); }, + LinkedHashMap::new + ) + ); + } + + /** * Returns all unassigned rotations * @param lock Lock which must be acquired by the caller */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java index d50d141d625..1ac82317695 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java @@ -2,15 +2,17 @@ package com.yahoo.vespa.hosted.controller.tls; import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.PeerAuthentication; +import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.hosted.controller.tls.config.TlsConfig; -import org.eclipse.jetty.util.ssl.SslContextFactory; import java.nio.file.Files; import java.nio.file.Paths; @@ -28,11 +30,11 @@ import java.util.concurrent.ConcurrentHashMap; * @author bjorncs */ @SuppressWarnings("unused") // Injected -public class ControllerSslContextFactoryProvider extends AbstractComponent implements SslContextFactoryProvider { +public class ControllerSslContextFactoryProvider extends TlsContextBasedProvider { private final KeyStore truststore; private final KeyStore keystore; - private final Map<Integer, SslContextFactory> sslContextFactories = new ConcurrentHashMap<>(); + private final Map<Integer, TlsContext> tlsContextMap = new ConcurrentHashMap<>(); @Inject public ControllerSslContextFactoryProvider(SecretStore secretStore, TlsConfig config) { @@ -50,24 +52,17 @@ public class ControllerSslContextFactoryProvider extends AbstractComponent imple } @Override - public SslContextFactory getInstance(String containerId, int port) { - return sslContextFactories.computeIfAbsent(port, this::createSslContextFactory); + protected TlsContext getTlsContext(String containerId, int port) { + return tlsContextMap.computeIfAbsent(port, this::createTlsContext); } - /** Create a SslContextFactory backed by an in-memory key and trust store */ - private SslContextFactory createSslContextFactory(int port) { - // TODO Use DefaultTlsContext to configure SslContextFactory (ensure that cipher/protocol configuration is same across all TLS endpoints). - - SslContextFactory.Server factory = new SslContextFactory.Server(); - if (port != 443) { - factory.setWantClientAuth(true); - } - factory.setTrustStore(truststore); - factory.setKeyStore(keystore); - factory.setKeyStorePassword(""); - factory.setExcludeProtocols("TLSv1.3"); // TLSv1.3 is broken is multiple OpenJDK 11 versions - factory.setEndpointIdentificationAlgorithm(null); // disable https hostname verification of clients (must be disabled when using Athenz x509 certificates) - return factory; + private TlsContext createTlsContext(int port) { + return new DefaultTlsContext( + new SslContextBuilder() + .withKeyStore(keystore, new char[0]) + .withTrustStore(truststore) + .build(), + port != 443 ? PeerAuthentication.WANT : PeerAuthentication.DISABLED); } /** Get private key from secret store **/ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index de31f1f67f9..c26f1879f6a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; @@ -23,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevisi import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; @@ -275,6 +277,8 @@ public class ControllerTest { @Test public void testDnsAliasRegistration() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -290,12 +294,42 @@ public class ControllerTest { for (Deployment deployment : deployments) { assertEquals("Rotation names are passed to config server in " + deployment.zone(), Set.of("rotation-id-01", - "app1--tenant1.global.vespa.oath.cloud", - "app1.tenant1.global.vespa.yahooapis.com", - "app1--tenant1.global.vespa.yahooapis.com"), + "app1--tenant1.global.vespa.oath.cloud"), tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone()))); } tester.flushDnsRequests(); + + assertEquals(1, tester.controllerTester().nameService().records().size()); + + var record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record.isPresent()); + assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); + assertEquals("rotation-fqdn-01.", record.get().data().asString()); + } + + @Test + public void testDnsAliasRegistrationLegacy() { + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .globalServiceId("foo") + .region("us-west-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage); + Collection<Deployment> deployments = tester.application(application.id()).deployments().values(); + assertFalse(deployments.isEmpty()); + for (Deployment deployment : deployments) { + assertEquals("Rotation names are passed to config server in " + deployment.zone(), + Set.of("rotation-id-01", + "app1--tenant1.global.vespa.oath.cloud", + "app1.tenant1.global.vespa.yahooapis.com", + "app1--tenant1.global.vespa.yahooapis.com"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone()))); + } + tester.flushDnsRequests(); assertEquals(3, tester.controllerTester().nameService().records().size()); Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); @@ -315,38 +349,134 @@ public class ControllerTest { } @Test - public void testRedirectLegacyDnsNames() { // TODO: Remove together with Flags.REDIRECT_LEGACY_DNS_NAMES + public void testDnsAliasRegistrationWithEndpoints() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) - .globalServiceId("foo") + .endpoint("foobar", "qrs", "us-west-1", "us-central-1") + .endpoint("default", "qrs", "us-west-1", "us-central-1") .region("us-west-1") .region("us-central-1") .build(); - ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.REDIRECT_LEGACY_DNS_NAMES.id(), true); + tester.deployCompletely(application, applicationPackage); + Collection<Deployment> deployments = tester.application(application.id()).deployments().values(); + assertFalse(deployments.isEmpty()); + for (Deployment deployment : deployments) { + assertEquals("Rotation names are passed to config server in " + deployment.zone(), + Set.of( + "rotation-id-01", + "rotation-id-02", + "app1--tenant1.global.vespa.oath.cloud", + "foobar--app1--tenant1.global.vespa.oath.cloud" + ), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone()))); + } + tester.flushDnsRequests(); + + assertEquals(2, tester.controllerTester().nameService().records().size()); + + var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record1.isPresent()); + assertEquals("app1--tenant1.global.vespa.oath.cloud", record1.get().name().asString()); + assertEquals("rotation-fqdn-02.", record1.get().data().asString()); + + var record2 = tester.controllerTester().findCname("foobar--app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record2.isPresent()); + assertEquals("foobar--app1--tenant1.global.vespa.oath.cloud", record2.get().name().asString()); + assertEquals("rotation-fqdn-01.", record2.get().data().asString()); + } + + @Test + public void testDnsAliasRegistrationWithChangingZones() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "qrs", "us-west-1", "us-central-1") + .region("us-west-1") + .region("us-central-1") + .build(); tester.deployCompletely(application, applicationPackage); - assertEquals(3, tester.controllerTester().nameService().records().size()); - Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("app1--tenant1.global.vespa.oath.cloud.", record.get().data().asString()); + assertEquals( + Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1"))) + ); - record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); + assertEquals( + Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-central-1"))) + ); - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("app1--tenant1.global.vespa.oath.cloud.", record.get().data().asString()); + + ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "qrs", "us-west-1") + .region("us-west-1") + .region("us-central-1") + .build(); + + tester.deployCompletely(application, applicationPackage2, BuildJob.defaultBuildNumber + 1); + + assertEquals( + Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1"))) + ); + + assertEquals( + Set.of(), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-central-1"))) + ); + + assertEquals(Set.of(RegionName.from("us-west-1")), tester.application(application.id()).assignedRotations().get(0).regions()); } @Test + public void testUnassignRotations() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "qrs", "us-west-1", "us-central-1") + .region("us-west-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage); + + ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage2, BuildJob.defaultBuildNumber + 1); + + + assertEquals( + List.of(AssignedRotation.fromStrings("qrs", "default", "rotation-id-01", Set.of())), + tester.application(application.id()).assignedRotations() + ); + + assertEquals( + Set.of(), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1"))) + ); + } + + @Test public void testUpdatesExistingDnsAlias() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + // Application 1 is deployed and deleted { Application app1 = tester.createApplication("app1", "tenant1", 1, 1L); @@ -358,16 +488,11 @@ public class ControllerTest { .build(); tester.deployCompletely(app1, applicationPackage); - assertEquals(3, tester.controllerTester().nameService().records().size()); - - Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); + assertEquals(1, tester.controllerTester().nameService().records().size()); - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); + Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record.isPresent()); - assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); + assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); // Application is deleted and rotation is unassigned @@ -408,22 +533,12 @@ public class ControllerTest { .region("us-central-1") .build(); tester.deployCompletely(app2, applicationPackage); - assertEquals(3, tester.controllerTester().nameService().records().size()); - - Optional<Record> record = tester.controllerTester().findCname("app2--tenant2.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app2--tenant2.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); + assertEquals(1, tester.controllerTester().nameService().records().size()); - record = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); + var record = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); assertTrue(record.isPresent()); assertEquals("app2--tenant2.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - - record = tester.controllerTester().findCname("app2.tenant2.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app2.tenant2.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); } // Application 1 is recreated, deployed and assigned a new rotation @@ -441,19 +556,15 @@ public class ControllerTest { assertEquals("rotation-id-02", app1.rotations().get(0).asString()); // DNS records are created for the newly assigned rotation - assertEquals(6, tester.controllerTester().nameService().records().size()); + assertEquals(2, tester.controllerTester().nameService().records().size()); - Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("rotation-fqdn-02.", record.get().data().asString()); - - record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); - assertTrue(record.isPresent()); - assertEquals("rotation-fqdn-02.", record.get().data().asString()); + var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record1.isPresent()); + assertEquals("rotation-fqdn-02.", record1.get().data().asString()); - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("rotation-fqdn-02.", record.get().data().asString()); + var record2 = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); + assertTrue(record2.isPresent()); + assertEquals("rotation-fqdn-01.", record2.get().data().asString()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index dbf983a5bab..d5935c752d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; -import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -29,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; -import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; @@ -100,7 +98,7 @@ public final class ControllerTester { this(new AthenzDbMock(), clock, new ConfigServerMock(new ZoneRegistryMock()), new ZoneRegistryMock(), new GitHubMock(), curatorDb, rotationsConfig, new MemoryNameService(), new ArtifactRepositoryMock(), new ApplicationStoreMock(), new MockBuildService(), - metricsService, new RoutingGeneratorMock(), new MockContactRetriever(), new MockIssueHandler(clock)); + metricsService, new RoutingGeneratorMock(), new MockContactRetriever()); } public ControllerTester(ManualClock clock) { @@ -125,7 +123,7 @@ public final class ControllerTester { MemoryNameService nameService, ArtifactRepositoryMock artifactRepository, ApplicationStoreMock appStoreMock, MockBuildService buildService, MetricsServiceMock metricsService, RoutingGeneratorMock routingGenerator, - MockContactRetriever contactRetriever, MockIssueHandler issueHandler) { + MockContactRetriever contactRetriever) { this.athenzDb = athenzDb; this.clock = clock; this.configServer = configServer; @@ -141,7 +139,7 @@ public final class ControllerTester { this.routingGenerator = routingGenerator; this.contactRetriever = contactRetriever; this.controller = createController(curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, - athenzDb, nameService, artifactRepository, appStoreMock, buildService, + athenzDb, artifactRepository, appStoreMock, buildService, metricsService, routingGenerator); // Make root logger use time from manual clock @@ -199,7 +197,7 @@ public final class ControllerTester { /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */ public final void createNewController() { controller = createController(curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, athenzDb, - nameService, artifactRepository, applicationStore, buildService, metricsService, + artifactRepository, applicationStore, buildService, metricsService, routingGenerator); } @@ -332,7 +330,7 @@ public final class ControllerTester { private static Controller createController(CuratorDb curator, RotationsConfig rotationsConfig, ConfigServerMock configServer, ManualClock clock, GitHubMock gitHub, ZoneRegistryMock zoneRegistryMock, - AthenzDbMock athensDb, MemoryNameService nameService, + AthenzDbMock athensDb, ArtifactRepository artifactRepository, ApplicationStore applicationStore, BuildService buildService, MetricsServiceMock metricsService, RoutingGenerator routingGenerator) { @@ -343,7 +341,6 @@ public final class ControllerTester { configServer, metricsService, routingGenerator, - new ChefMock(), clock, new AthenzFacade(new AthenzClientFactoryMock(athensDb)), artifactRepository, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index 16b875c1892..f5047a82e2f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -66,6 +66,50 @@ public class EndpointTest { } @Test + public void test_global_endpoints_with_endpoint_id() { + final var endpointId = EndpointId.default_(); + + Map<String, Endpoint> tests = Map.of( + // Legacy endpoint + "http://a1.t1.global.vespa.yahooapis.com:4080/", + Endpoint.of(app1).named(endpointId).on(Port.plain(4080)).legacy().in(SystemName.main), + + // Legacy endpoint with TLS + "https://a1--t1.global.vespa.yahooapis.com:4443/", + Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).legacy().in(SystemName.main), + + // Main endpoint + "https://a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).in(SystemName.main), + + // Main endpoint in CD + "https://cd--a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).in(SystemName.cd), + + // Main endpoint with direct routing and default TLS port + "https://a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint with custom rotation name + "https://r1.a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance in default rotation + "https://a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance with custom rotation name + "https://r2.a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint in public system + "https://a1.t1.global.public.vespa.oath.cloud/", + Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.Public) + ); + tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); + } + + @Test public void test_zone_endpoints() { ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); // Always default for non-direct routing ZoneId prodZone = ZoneId.from("prod", "us-north-1"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 83b95ccc8b0..6635547e9be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -17,6 +17,9 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.OptionalInt; import java.util.StringJoiner; import java.util.zip.ZipEntry; @@ -35,6 +38,7 @@ public class ApplicationPackageBuilder { private final StringJoiner notifications = new StringJoiner("/>\n <email ", "<notifications>\n <email ", "/>\n</notifications>\n").setEmptyValue(""); + private final StringBuilder endpointsBody = new StringBuilder(); private OptionalInt majorVersion = OptionalInt.empty(); private String upgradePolicy = null; @@ -63,6 +67,18 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder endpoint(String endpointId, String containerId, String... regions) { + endpointsBody.append(" <endpoint"); + endpointsBody.append(" id='").append(endpointId).append("'"); + endpointsBody.append(" container-id='").append(containerId).append("'"); + endpointsBody.append(">\n"); + for (var region : regions) { + endpointsBody.append(" <region>").append(region).append("</region>\n"); + } + endpointsBody.append(" </endpoint>\n"); + return this; + } + public ApplicationPackageBuilder region(RegionName regionName) { return region(regionName.value()); } @@ -157,7 +173,11 @@ public class ApplicationPackageBuilder { xml.append(environmentBody); xml.append(" </"); xml.append(environment.value()); - xml.append(">\n</deployment>"); + xml.append(">\n"); + xml.append(" <endpoints>\n"); + xml.append(endpointsBody); + xml.append(" </endpoints>\n"); + xml.append("</deployment>"); return xml.toString().getBytes(StandardCharsets.UTF_8); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index cdbc45c4d8f..a89c5988396 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -45,6 +45,7 @@ import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; /** * @author mortent @@ -226,7 +227,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationNames, - List<ContainerEndpoint> containerEndpoints, ApplicationCertificate applicationCertificate, byte[] content) { + Set<ContainerEndpoint> containerEndpoints, ApplicationCertificate applicationCertificate, byte[] content) { lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null); if (prepareException != null) { RuntimeException prepareException = this.prepareException; @@ -238,7 +239,13 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer if (nodeRepository().list(deployment.zoneId(), deployment.applicationId()).isEmpty()) provision(deployment.zoneId(), deployment.applicationId()); - this.rotationNames.put(deployment, Set.copyOf(rotationNames)); + this.rotationNames.put( + deployment, + Stream.concat( + containerEndpoints.stream().flatMap(e -> e.names().stream()), + rotationNames.stream() + ).collect(Collectors.toSet()) + ); return () -> { Application application = applications.get(deployment.applicationId()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 58f35c0ac05..148be3f258e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -1,38 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.SystemName; -import com.yahoo.test.ManualClock; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; -import com.yahoo.vespa.hosted.controller.integration.MetricsMock.MapContext; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Clock; import java.time.Duration; -import java.time.Instant; -import java.util.Map; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; @@ -40,39 +25,13 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; /** * @author mortent */ public class MetricsReporterTest { - private static final Path testData = Paths.get("src/test/resources/"); - - private MetricsMock metrics; - - @Before - public void before() { - metrics = new MetricsMock(); - } - - @Test - public void test_chef_metrics() { - Clock clock = new ManualClock(Instant.ofEpochSecond(1475497913)); - ControllerTester tester = new ControllerTester(); - MetricsReporter metricsReporter = createReporter(clock, tester.controller(), metrics, SystemName.cd); - metricsReporter.maintain(); - assertEquals(2, metrics.getMetrics().size()); - - Map<MapContext, Map<String, Number>> hostMetrics = getMetricsByHost("fake-node.test"); - assertEquals(1, hostMetrics.size()); - Map.Entry<MapContext, Map<String, Number>> metricEntry = hostMetrics.entrySet().iterator().next(); - MapContext metricContext = metricEntry.getKey(); - assertDimension(metricContext, "tenantName", "ciintegrationtests"); - assertDimension(metricContext, "app", "restart.default"); - assertDimension(metricContext, "zone", "prod.cd-us-east-1"); - assertEquals(727, metricEntry.getValue().get(MetricsReporter.CONVERGENCE_METRIC).longValue()); - } + private final MetricsMock metrics = new MetricsMock(); @Test public void test_deployment_fail_ratio() { @@ -81,7 +40,7 @@ public class MetricsReporterTest { .environment(Environment.prod) .region("us-west-1") .build(); - MetricsReporter metricsReporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter metricsReporter = createReporter(tester.controller()); metricsReporter.maintain(); assertEquals(0.0, metrics.getMetric(MetricsReporter.DEPLOYMENT_FAIL_METRIC)); @@ -108,14 +67,6 @@ public class MetricsReporterTest { } @Test - public void test_chef_metrics_omit_zone_when_unknown() { - ControllerTester tester = new ControllerTester(); - String hostname = "fake-node2.test"; - MapContext metricContext = getMetricContextByHost(tester.controller(), hostname); - assertNull(metricContext.getDimensions().get("zone")); - } - - @Test public void test_deployment_average_duration() { DeploymentTester tester = new DeploymentTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -123,7 +74,7 @@ public class MetricsReporterTest { .region("us-west-1") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application app = tester.createApplication("app1", "tenant1", 1, 11L); tester.deployCompletely(app, applicationPackage); @@ -165,7 +116,7 @@ public class MetricsReporterTest { .region("us-west-1") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application app = tester.createApplication("app1", "tenant1", 1, 11L); // Initial deployment without failures @@ -216,7 +167,7 @@ public class MetricsReporterTest { .region("us-west-1") .region("us-east-3") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application application = tester.createApplication("app1", "tenant1", 1, 11L); tester.configServer().generateWarnings(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1")), 3); tester.configServer().generateWarnings(new DeploymentId(application.id(), ZoneId.from("prod", "us-east-3")), 4); @@ -231,7 +182,7 @@ public class MetricsReporterTest { ApplicationVersion version = tester.deployNewSubmission(); assertEquals(1000, version.buildTime().get().toEpochMilli()); - MetricsReporter reporter = createReporter(tester.tester().controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.tester().controller()); reporter.maintain(); assertEquals(tester.clock().instant().getEpochSecond() - 1, getMetric(MetricsReporter.DEPLOYMENT_BUILD_AGE_SECONDS, tester.app())); @@ -246,7 +197,7 @@ public class MetricsReporterTest { .region("us-west-1") .region("us-east-3") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application application = tester.createApplication("app1", "tenant1", 1, 11L); reporter.maintain(); assertEquals("Queue is empty initially", 0, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); @@ -279,43 +230,8 @@ public class MetricsReporterTest { .orElseThrow(() -> new RuntimeException("Expected metric to exist for " + application.id())); } - private MetricsReporter createReporter(Controller controller, MetricsMock metricsMock, SystemName system) { - return createReporter(controller.clock(), controller, metricsMock, system); - } - - private MetricsReporter createReporter(Clock clock, Controller controller, MetricsMock metricsMock, - SystemName system) { - ChefMock chef = new ChefMock(); - PartialNodeResult result; - try { - result = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .readValue(testData.resolve("chef_output.json").toFile(), PartialNodeResult.class); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - chef.addPartialResult(result.rows); - return new MetricsReporter(controller, metricsMock, chef, clock, new JobControl(new MockCuratorDb()), system); - } - - private Map<MapContext, Map<String, Number>> getMetricsByHost(String hostname) { - return metrics.getMetrics((dimensions) -> hostname.equals(dimensions.get("host"))); - } - - private MapContext getMetricContextByHost(Controller controller, String hostname) { - MetricsReporter metricsReporter = createReporter(controller, metrics, SystemName.main); - metricsReporter.maintain(); - - assertFalse(metrics.getMetrics().isEmpty()); - - Map<MapContext, Map<String, Number>> metrics = getMetricsByHost(hostname); - assertEquals(1, metrics.size()); - Map.Entry<MapContext, Map<String, Number>> metricEntry = metrics.entrySet().iterator().next(); - return metricEntry.getKey(); - } - - private static void assertDimension(MapContext metricContext, String dimensionName, String expectedValue) { - assertEquals(expectedValue, metricContext.getDimensions().get(dimensionName)); + private MetricsReporter createReporter(Controller controller) { + return new MetricsReporter(controller, metrics, new JobControl(new MockCuratorDb())); } private static String appDimension(Application application) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 347fe6064df..7b39b0d53a4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; @@ -46,6 +47,7 @@ import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; +import java.util.Set; import java.util.TreeMap; import static com.yahoo.config.provision.SystemName.main; @@ -119,7 +121,7 @@ public class ApplicationSerializerTest { OptionalInt.of(7), new MetricsService.ApplicationMetrics(0.5, 0.9), Optional.of("-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"), - List.of(new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.default_(), new RotationId("my-rotation"))), + List.of(AssignedRotation.fromStrings("foo", "default", "my-rotation", Set.of())), rotationStatus, Optional.of(new ApplicationCertificate("vespa.certificate"))); @@ -258,6 +260,11 @@ public class ApplicationSerializerTest { final var applicationJson = Files.readAllBytes(testData.resolve("complete-application.json")); final var slime = SlimeUtils.jsonToSlime(applicationJson); + final var regions = Set.of( + RegionName.from("us-east-3"), + RegionName.from("us-west-1") + ); + // Add the necessary fields to the Slime representation of the application final var cursor = slime.get(); cursor.setString("rotation", "single-rotation"); @@ -275,11 +282,11 @@ public class ApplicationSerializerTest { // Parse and test the output from parsing contains both legacy rotation and multiple rotations final var application = applicationSerializer.fromSlime(slime); + // Since only one AssignedEndpoint can be "default", we make sure that we are ignoring the + // multiple-rotation entries as the globalServiceId will override them assertEquals( List.of( new RotationId("single-rotation"), - new RotationId("multiple-rotation-1"), - new RotationId("multiple-rotation-2"), new RotationId("assigned-rotation") ), application.rotations() @@ -289,12 +296,13 @@ public class ApplicationSerializerTest { Optional.of(new RotationId("single-rotation")), application.legacyRotation() ); + // The same goes here for AssignedRotations with "default" EndpointId as in the .rotations() test above. + // Note that we are only using Set.of() on "assigned-rotation" because in this test we do not have access + // to a deployment.xml that describes the zones a rotation should map to. assertEquals( List.of( - new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("single-rotation")), - new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("multiple-rotation-1")), - new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("multiple-rotation-2")), - new AssignedRotation(new ClusterSpec.Id("foobar"), EndpointId.of("nice-endpoint"), new RotationId("assigned-rotation")) + new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("single-rotation"), regions), + new AssignedRotation(new ClusterSpec.Id("foobar"), EndpointId.of("nice-endpoint"), new RotationId("assigned-rotation"), Set.of()) ), application.assignedRotations() ); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 53476a2e42f..797d2b9aa0e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -62,7 +62,6 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock'/>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 16fd10277d2..29931a1f626 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -16,7 +16,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.test.ManualClock; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; @@ -61,7 +60,6 @@ import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import org.junit.Before; @@ -243,7 +241,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/test/region/us-east-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in test.us-east-1\"}"); controllerTester.jobCompletion(JobType.systemTest) .application(id) @@ -257,7 +255,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/staging/region/us-east-3"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in staging.us-east-3\"}"); controllerTester.jobCompletion(JobType.stagingTest) .application(id) .projectId(screwdriverProjectId) @@ -369,7 +367,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", DELETE) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), - ""); + "{\"message\":\"Deleted application tenant2.application2\"}"); // Set version 6.1 to broken to change compile version for. controllerTester.upgrader().overrideConfidence(Version.fromString("6.1"), VespaVersion.Confidence.broken); @@ -482,27 +480,27 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}"); // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}"); // POST a 'restart application' in staging environment command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in staging.us-central-1\"}"); // POST a 'restart application' in staging test command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in test.us-central-1\"}"); // POST a 'restart application' in staging dev command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in dev.us-central-1\"}"); // POST a 'restart application' command with a host filter (other filters not supported yet) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart?hostname=host1", POST) @@ -532,18 +530,18 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE (deactivate) a deployment - dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1", DELETE) .userIdentity(USER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/dev/region/us-west-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in dev.us-west-1\"}"); // DELETE (deactivate) a deployment - prod tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in prod.us-central-1\"}"); // DELETE (deactivate) a deployment is idempotent tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in prod.us-central-1\"}"); // POST an application package to start a deployment to dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1", POST) @@ -663,7 +661,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE).userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), - ""); + "{\"message\":\"Deleted application tenant1.application1.instance1\"}"); // DELETE a tenant tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE).userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), @@ -992,7 +990,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), - ""); + "{\"message\":\"Deleted application tenant1.application1.instance1\"}"); // DELETE application again - should produce 404 tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .oktaAccessToken(OKTA_AT) @@ -1091,7 +1089,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) .userIdentity(authorizedUser) .oktaAccessToken(OKTA_AT), - "", + "{\"message\":\"Deleted application tenant1.application1\"}", 200); // Updating a tenant for an Athens domain the user is not admin for is disallowed @@ -1530,7 +1528,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request(testPath, DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated " + testPath.replaceFirst("/application/v4/", "")); + "{\"message\":\"Deactivated " + application + " in test.us-east-1\"}"); controllerTester.jobCompletion(JobType.systemTest) .application(application) .projectId(projectId) @@ -1545,7 +1543,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request(stagingPath, DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated " + stagingPath.replaceFirst("/application/v4/", "")); + "{\"message\":\"Deactivated " + application + " in staging.us-east-3\"}"); controllerTester.jobCompletion(JobType.stagingTest) .application(application) .projectId(projectId) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index c845d31a5fc..93b6138d987 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -1,25 +1,31 @@ { + "lastVersions": {}, + "deploying": {}, "deployments": [], "jobs": {}, "devJobs": { "dev-us-east-1": { - "id": 1, - "status": "success", - "start": 0, - "end": 0, - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "unknown" - }, - "steps": { - "deployReal": "succeeded", - "installReal": "succeeded" - }, - "tasks": { - "deploy": "succeeded", - "install": "succeeded" - }, - "log": "https://some.url:43/root/dev-us-east-1/run/1" + "runs": [ + { + "id": 1, + "status": "success", + "start": 0, + "end": 0, + "wantedPlatform": "6.1", + "wantedApplication": { + "hash": "unknown" + }, + "steps": { + "deployReal": "succeeded", + "installReal": "succeeded" + }, + "tasks": { + "deploy": "succeeded", + "install": "succeeded" + }, + "log": "https://some.url:43/root/dev-us-east-1/run/1" + } + ] } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json deleted file mode 100644 index fef3cf6a372..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "application": "tenant1:application1:default", - "zone": "dev.us-east-1", - "system": "main", - "endpoints": { - "dev.us-east-1": [ - "http://old-endpoint.vespa.yahooapis.com:4080" - ], - "prod.us-central-1": [ - "http://old-endpoint.vespa.yahooapis.com:4080" - ] - }, - "zoneEndpoints": { - "dev.us-east-1": { - "default": "http://old-endpoint.vespa.yahooapis.com:4080" - }, - "prod.us-central-1": { - "default": "http://old-endpoint.vespa.yahooapis.com:4080" - } - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 59f63f0472a..d0e9ae77965 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -137,7 +137,7 @@ public class UserApiTest extends ControllerContainerCloudTest { // DELETE an application is available to application admins. tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app", DELETE) .roles(Set.of(Role.applicationAdmin(id.tenant(), id.application()))), - ""); + "{\"message\":\"Deleted application my-tenant.my-app\"}"); // DELETE a tenant role is available to tenant admins. tester.assertResponse(request("/user/v1/tenant/my-tenant", DELETE) diff --git a/controller-server/src/test/resources/chef_output.json b/controller-server/src/test/resources/chef_output.json deleted file mode 100644 index 257065f7b5b..00000000000 --- a/controller-server/src/test/resources/chef_output.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "total": 1, - "start": 0, - "rows": [ - { - "url": "https://chef-server.test/organizations/vespa/nodes/fake-node.test", - "data": { - "fqdn": "fake-node.test", - "ohai_time": 1475497186.68962, - "tenant": "ciintegrationtests", - "application": "restart", - "instance": "default", - "zone": "cd_cd-us-east-1_prod", - "system": "cd", - "environment": "prod", - "region": "cd-us-east-1" - } - }, - { - "url": "https://chef-server.test/organizations/vespa/nodes/fake-node2.test", - "data": { - "fqdn": "fake-node2.test", - "ohai_time": 1475497186.68962, - "tenant": null, - "application": null, - "instance": null, - "zone": null, - "system": null, - "environment": null, - "region": null - } - } - ] -} diff --git a/controller-server/src/test/resources/job-grandparent.json b/controller-server/src/test/resources/job-grandparent.json deleted file mode 100644 index 63602bed146..00000000000 --- a/controller-server/src/test/resources/job-grandparent.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "duration": 720000, - "causes": [] -} diff --git a/controller-server/src/test/resources/job-parent.json b/controller-server/src/test/resources/job-parent.json deleted file mode 100644 index 88d50de394f..00000000000 --- a/controller-server/src/test/resources/job-parent.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "duration": 1200000, - "causes": [ - { - "upstreamBuild": 231, - "upstreamProject": "3-v3-job-grandparent" - } - ] -} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java index 46a0f9b9b10..590ef207e3f 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java @@ -1,22 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.dockerapi.metrics; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.TreeMap; import java.util.stream.Collectors; /** * @author freva */ public class DimensionMetrics { - private static final ObjectMapper objectMapper = new ObjectMapper(); - private static final Map<String, Object> routing = Map.of("yamas", Map.of("namespaces", List.of("Vespa"))); private final String application; private final Dimensions dimensions; @@ -30,17 +23,6 @@ public class DimensionMetrics { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - public String toSecretAgentReport() throws JsonProcessingException { - Map<String, Object> report = new TreeMap<>(); - report.put("application", application); - report.put("dimensions", new TreeMap<>(dimensions.asMap())); - report.put("metrics", new TreeMap<>(metrics)); - report.put("routing", routing); - report.put("timestamp", System.currentTimeMillis() / 1000); - - return objectMapper.writeValueAsString(report); - } - public String getApplication() { return application; } diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java index 89a5134943a..5881267c252 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java @@ -18,6 +18,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; public class SingleValueReader { + public static final String UPDATE_ASSIGN = "assign"; public static final String UPDATE_INCREMENT = "increment"; public static final String UPDATE_DECREMENT = "decrement"; diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index e0159febdfe..d91e35f2d63 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -29,6 +29,7 @@ vespa_define_module( src/tests/tensor/dense_add_dimension_optimizer src/tests/tensor/dense_dot_product_function src/tests/tensor/dense_fast_rename_optimizer + src/tests/tensor/dense_generic_join src/tests/tensor/dense_inplace_join_function src/tests/tensor/dense_inplace_map_function src/tests/tensor/dense_remove_dimension_optimizer @@ -39,7 +40,6 @@ vespa_define_module( src/tests/tensor/tensor_add_operation src/tests/tensor/tensor_address src/tests/tensor/tensor_conformance - src/tests/tensor/tensor_mapper src/tests/tensor/tensor_modify_operation src/tests/tensor/tensor_remove_operation src/tests/tensor/tensor_serialization diff --git a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp index d970b9dad30..356625417d8 100644 --- a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp @@ -102,7 +102,7 @@ EvalFixture::ParamRepo make_params() { .add("v04_y3", spec({y(3)}, MyVecSeq(10))) .add("v05_x5", spec({x(5)}, MyVecSeq(6.0))) .add("v06_x5", spec({x(5)}, MyVecSeq(7.0))) - .add("v07_x5f", spec({x(5)}, MyVecSeq(7.0)), "tensor<float>(x[5])") + .add("v07_x5f", spec(float_cells({x(5)}), MyVecSeq(7.0))) .add("m01_x3y3", spec({x(3),y(3)}, MyVecSeq(1.0))) .add("m02_x3y3", spec({x(3),y(3)}, MyVecSeq(2.0))); } diff --git a/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp b/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp index 773381b4c77..4995ea89735 100644 --- a/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp +++ b/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp @@ -25,7 +25,7 @@ const TensorEngine &prod_engine = DefaultTensorEngine::ref(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() .add("x5", spec({x(5)}, N())) - .add("x5f", spec({x(5)}, N()), "tensor<float>(x[5])") + .add("x5f", spec(float_cells({x(5)}), N())) .add("x_m", spec({x({"a", "b", "c"})}, N())) .add("x5y3", spec({x(5),y(3)}, N())); } diff --git a/eval/src/tests/tensor/dense_generic_join/CMakeLists.txt b/eval/src/tests/tensor/dense_generic_join/CMakeLists.txt new file mode 100644 index 00000000000..1fbb35cb2b8 --- /dev/null +++ b/eval/src/tests/tensor/dense_generic_join/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_dense_generic_join_test_app TEST + SOURCES + dense_generic_join_test.cpp + DEPENDS + vespaeval +) +vespa_add_test(NAME eval_dense_generic_join_test_app COMMAND eval_dense_generic_join_test_app) diff --git a/eval/src/tests/tensor/dense_generic_join/dense_generic_join_test.cpp b/eval/src/tests/tensor/dense_generic_join/dense_generic_join_test.cpp new file mode 100644 index 00000000000..395437e13dd --- /dev/null +++ b/eval/src/tests/tensor/dense_generic_join/dense_generic_join_test.cpp @@ -0,0 +1,127 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/eval/simple_tensor.h> +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/dense/typed_dense_tensor_builder.h> +#include <vespa/eval/tensor/dense/dense_tensor.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/eval_fixture.h> + +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; +using namespace vespalib::tensor; +using namespace vespalib::eval::tensor_function; + +const TensorEngine &prod_engine = DefaultTensorEngine::ref(); + +double seq_value = 0.0; + +struct GlobalSequence : public Sequence { + GlobalSequence() {} + double operator[](size_t) const override { + seq_value += 1.0; + return seq_value; + } + ~GlobalSequence() {} +}; +GlobalSequence seq; + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add("con_x5_A", spec({x(5) }, seq)) + .add("con_x5y3_B", spec({x(5),y(3) }, seq)) + .add("con_x5z4_C", spec({x(5), z(4)}, seq)) + .add("con_x5y3z4_D", spec({x(5),y(3),z(4)}, seq)) + .add("con_y3_E", spec({ y(3) }, seq)) + .add("con_y3z4_F", spec({ y(3),z(4)}, seq)) + .add("con_z4_G", spec({ z(4)}, seq)) + .add("con_x5f_H", spec({x(5) }, seq), "tensor<float>(x[5])") + .add("con_x5y3_I", spec({x(5),y(3) }, seq), "tensor<float>(x[5],y[3])") + .add("con_x5z4_J", spec({x(5), z(4)}, seq), "tensor<float>(x[5],z[4])") + .add("con_x5y3z4_K", spec({x(5),y(3),z(4)}, seq), "tensor<float>(x[5],y[3],z[4])") + .add("con_y3_L", spec({ y(3) }, seq), "tensor<float>(y[3])") + .add("con_y3z4_M", spec({ y(3),z(4)}, seq), "tensor<float>(y[3],z[4])))") + .add("con_z4_N", spec({ z(4)}, seq), "tensor<float>(z[4]))") + .add("con_y2", spec({y(5)}, seq)) + .add("con_y2f", spec({y(5)}, seq), "tensor<float>(y[2]))"); +} +EvalFixture::ParamRepo param_repo = make_params(); + +void verify_equal(const vespalib::string &expr) { + EvalFixture fixture(prod_engine, expr, param_repo, true, true); + EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); +} + + +TEST("require that non-overlapping dense join works") { + TEST_DO(verify_equal("con_x5_A-con_y3_E")); + TEST_DO(verify_equal("con_x5_A+con_y3_E")); + TEST_DO(verify_equal("con_x5_A*con_y3_E")); + + TEST_DO(verify_equal("con_x5_A-con_y3z4_F")); + TEST_DO(verify_equal("con_x5_A+con_y3z4_F")); + TEST_DO(verify_equal("con_x5_A*con_y3z4_F")); + + TEST_DO(verify_equal("con_x5_A-con_z4_G")); + TEST_DO(verify_equal("con_x5_A+con_z4_G")); + TEST_DO(verify_equal("con_x5_A*con_z4_G")); + + TEST_DO(verify_equal("con_x5y3_B-con_z4_G")); + TEST_DO(verify_equal("con_x5y3_B+con_z4_G")); + TEST_DO(verify_equal("con_x5y3_B*con_z4_G")); + + TEST_DO(verify_equal("con_y3_E-con_z4_G")); + TEST_DO(verify_equal("con_y3_E+con_z4_G")); + TEST_DO(verify_equal("con_y3_E*con_z4_G")); +} + +TEST("require that overlapping dense join works") { + TEST_DO(verify_equal("con_x5_A-con_x5y3_B")); + TEST_DO(verify_equal("con_x5_A+con_x5y3_B")); + TEST_DO(verify_equal("con_x5_A*con_x5y3_B")); + + TEST_DO(verify_equal("con_x5_A-con_x5z4_C")); + TEST_DO(verify_equal("con_x5_A+con_x5z4_C")); + TEST_DO(verify_equal("con_x5_A*con_x5z4_C")); + + TEST_DO(verify_equal("con_x5y3_B-con_y3_E")); + TEST_DO(verify_equal("con_x5y3_B+con_y3_E")); + TEST_DO(verify_equal("con_x5y3_B*con_y3_E")); + + TEST_DO(verify_equal("con_x5y3_B-con_y3z4_F")); + TEST_DO(verify_equal("con_x5y3_B+con_y3z4_F")); + TEST_DO(verify_equal("con_x5y3_B*con_y3z4_F")); + + TEST_DO(verify_equal("con_x5y3z4_D-con_x5y3_B")); + TEST_DO(verify_equal("con_x5y3z4_D+con_x5y3_B")); + TEST_DO(verify_equal("con_x5y3z4_D*con_x5y3_B")); + + TEST_DO(verify_equal("con_x5y3z4_D-con_x5z4_C")); + TEST_DO(verify_equal("con_x5y3z4_D+con_x5z4_C")); + TEST_DO(verify_equal("con_x5y3z4_D*con_x5z4_C")); + + TEST_DO(verify_equal("con_x5y3z4_D-con_y3z4_F")); + TEST_DO(verify_equal("con_x5y3z4_D+con_y3z4_F")); + TEST_DO(verify_equal("con_x5y3z4_D*con_y3z4_F")); + + TEST_DO(verify_equal("con_x5y3z4_D-con_y3z4_F")); + TEST_DO(verify_equal("con_x5y3z4_D+con_y3z4_F")); + TEST_DO(verify_equal("con_x5y3z4_D*con_y3z4_F")); + + TEST_DO(verify_equal("con_y3_E-con_y3z4_F")); + TEST_DO(verify_equal("con_y3_E+con_y3z4_F")); + TEST_DO(verify_equal("con_y3_E*con_y3z4_F")); + + TEST_DO(verify_equal("con_y3z4_F-con_z4_G")); + TEST_DO(verify_equal("con_y3z4_F+con_z4_G")); + TEST_DO(verify_equal("con_y3z4_F*con_z4_G")); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp b/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp index c9e581e6b21..083ed1c7071 100644 --- a/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp +++ b/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp @@ -45,8 +45,8 @@ EvalFixture::ParamRepo make_params() { .add_mutable("mut_x5_A", spec({x(5)}, seq)) .add_mutable("mut_x5_B", spec({x(5)}, seq)) .add_mutable("mut_x5_C", spec({x(5)}, seq)) - .add_mutable("mut_x5f_D", spec({x(5)}, seq), "tensor<float>(x[5])") - .add_mutable("mut_x5f_E", spec({x(5)}, seq), "tensor<float>(x[5])") + .add_mutable("mut_x5f_D", spec(float_cells({x(5)}), seq)) + .add_mutable("mut_x5f_E", spec(float_cells({x(5)}), seq)) .add_mutable("mut_x5y3_A", spec({x(5),y(3)}, seq)) .add_mutable("mut_x5y3_B", spec({x(5),y(3)}, seq)) .add_mutable("mut_x_sparse", spec({x({"a", "b", "c"})}, seq)); diff --git a/eval/src/tests/tensor/dense_inplace_map_function/dense_inplace_map_function_test.cpp b/eval/src/tests/tensor/dense_inplace_map_function/dense_inplace_map_function_test.cpp index 36ebdec028b..314d3a6186c 100644 --- a/eval/src/tests/tensor/dense_inplace_map_function/dense_inplace_map_function_test.cpp +++ b/eval/src/tests/tensor/dense_inplace_map_function/dense_inplace_map_function_test.cpp @@ -26,7 +26,7 @@ EvalFixture::ParamRepo make_params() { .add("x5", spec({x(5)}, N())) .add_mutable("_d", spec(5.0)) .add_mutable("_x5", spec({x(5)}, N())) - .add_mutable("_x5f", spec({x(5)}, N()), "tensor<float>(x[5])") + .add_mutable("_x5f", spec(float_cells({x(5)}), N())) .add_mutable("_x5y3", spec({x(5),y(3)}, N())) .add_mutable("_x_m", spec({x({"a", "b", "c"})}, N())); } diff --git a/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp b/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp index 65208aedb4b..7856775ae30 100644 --- a/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp +++ b/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp @@ -25,7 +25,7 @@ const TensorEngine &prod_engine = DefaultTensorEngine::ref(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() .add("x1y5z1", spec({x(1),y(5),z(1)}, N())) - .add("x1y5z1f", spec({x(1),y(5),z(1)}, N()), "tensor<float>(x[1],y[5],z[1])") + .add("x1y5z1f", spec(float_cells({x(1),y(5),z(1)}), N())) .add("x1y1z1", spec({x(1),y(1),z(1)}, N())) .add("x1y5z_m", spec({x(1),y(5),z({"a"})}, N())); } diff --git a/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp b/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp index 7e6b933d7c6..0533b1c92b7 100644 --- a/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp +++ b/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp @@ -13,11 +13,9 @@ using namespace vespalib::eval; using namespace vespalib::tensor; using namespace vespalib; -using CellsRef = DenseTensorView::CellsRef; - const TensorEngine &engine = DefaultTensorEngine::ref(); -CellsRef getCellsRef(const eval::Value &value) { +TypedCells getCellsRef(const eval::Value &value) { return static_cast<const DenseTensorView &>(value).cellsRef(); } @@ -58,8 +56,8 @@ TEST_F("require that DenseReplaceTypeFunction works as expected", Fixture()) { f1.mock_child.is_mutable = false; EXPECT_EQUAL(f1.my_fun.result_is_mutable(), false); EXPECT_EQUAL(&f1.children[0].get().get(), &f1.mock_child); - EXPECT_EQUAL(getCellsRef(f1.state.stack[0]).begin(), getCellsRef(*f1.my_value).begin()); - EXPECT_EQUAL(getCellsRef(f1.state.stack[0]).end(), getCellsRef(*f1.my_value).end()); + EXPECT_EQUAL(getCellsRef(f1.state.stack[0]).data, getCellsRef(*f1.my_value).data); + EXPECT_EQUAL(getCellsRef(f1.state.stack[0]).size, getCellsRef(*f1.my_value).size); EXPECT_EQUAL(f1.state.stack[0].get().type(), f1.new_type); fprintf(stderr, "%s\n", f1.my_fun.as_string().c_str()); } diff --git a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp index 8045958d9ba..335aa4791a4 100644 --- a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp @@ -38,13 +38,13 @@ EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() .add("y1", spec({y(1)}, MyVecSeq())) .add("y3", spec({y(3)}, MyVecSeq())) - .add("y3f", spec({y(3)}, MyVecSeq()), "tensor<float>(y[3])") + .add("y3f", spec(float_cells({y(3)}), MyVecSeq())) .add("y5", spec({y(5)}, MyVecSeq())) .add("y16", spec({y(16)}, MyVecSeq())) .add("x1y1", spec({x(1),y(1)}, MyMatSeq())) .add("y1z1", spec({y(1),z(1)}, MyMatSeq())) .add("x2y3", spec({x(2),y(3)}, MyMatSeq())) - .add("x2y3f", spec({x(2),y(3)}, MyMatSeq()), "tensor<float>(x[2],y[3])") + .add("x2y3f", spec(float_cells({x(2),y(3)}), MyMatSeq())) .add("x2z3", spec({x(2),z(3)}, MyMatSeq())) .add("y3z2", spec({y(3),z(2)}, MyMatSeq())) .add("x8y5", spec({x(8),y(5)}, MyMatSeq())) diff --git a/eval/src/tests/tensor/direct_dense_tensor_builder/direct_dense_tensor_builder_test.cpp b/eval/src/tests/tensor/direct_dense_tensor_builder/direct_dense_tensor_builder_test.cpp index a0b7b60e2da..3f4641ed2ee 100644 --- a/eval/src/tests/tensor/direct_dense_tensor_builder/direct_dense_tensor_builder_test.cpp +++ b/eval/src/tests/tensor/direct_dense_tensor_builder/direct_dense_tensor_builder_test.cpp @@ -2,31 +2,36 @@ #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/eval/tensor/dense/direct_dense_tensor_builder.h> +#include <vespa/eval/tensor/dense/typed_dense_tensor_builder.h> #include <vespa/vespalib/util/exceptions.h> using namespace vespalib::tensor; using vespalib::IllegalArgumentException; -using Builder = DirectDenseTensorBuilder; +using BuilderDbl = TypedDenseTensorBuilder<double>; +using BuilderFlt = TypedDenseTensorBuilder<float>; using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; using vespalib::ConstArrayRef; -template <typename T> std::vector<T> make_vector(const ConstArrayRef<T> &ref) { - std::vector<T> vec; - for (const T &t: ref) { - vec.push_back(t); +struct CallMakeVector { + template <typename T> + static std::vector<double> call(const ConstArrayRef<T> &ref) { + std::vector<double> result; + result.reserve(ref.size()); + for (T v : ref) { + result.push_back(v); + } + return result; } - return vec; -} +}; void assertTensor(const vespalib::string &type_spec, - const DenseTensor::Cells &expCells, + const std::vector<double> &expCells, const Tensor &tensor) { - const DenseTensor &realTensor = dynamic_cast<const DenseTensor &>(tensor); + const DenseTensorView &realTensor = dynamic_cast<const DenseTensorView &>(tensor); EXPECT_EQUAL(ValueType::from_spec(type_spec), realTensor.type()); - EXPECT_EQUAL(expCells, make_vector(realTensor.cellsRef())); + EXPECT_EQUAL(expCells, dispatch_1<CallMakeVector>(realTensor.cellsRef())); } void assertTensorSpec(const TensorSpec &expSpec, const Tensor &tensor) { @@ -35,7 +40,7 @@ void assertTensorSpec(const TensorSpec &expSpec, const Tensor &tensor) { } Tensor::UP build1DTensor() { - Builder builder(ValueType::from_spec("tensor(x[3])")); + BuilderDbl builder(ValueType::from_spec("tensor(x[3])")); builder.insertCell(0, 10); builder.insertCell(1, 11); builder.insertCell(2, 12); @@ -55,7 +60,7 @@ TEST("require that 1d tensor can be converted to tensor spec") { } Tensor::UP build2DTensor() { - Builder builder(ValueType::from_spec("tensor(x[3],y[2])")); + BuilderDbl builder(ValueType::from_spec("tensor(x[3],y[2])")); builder.insertCell({0, 0}, 10); builder.insertCell({0, 1}, 11); builder.insertCell({1, 0}, 12); @@ -81,7 +86,7 @@ TEST("require that 2d tensor can be converted to tensor spec") { } TEST("require that 3d tensor can be constructed") { - Builder builder(ValueType::from_spec("tensor(x[3],y[2],z[2])")); + BuilderDbl builder(ValueType::from_spec("tensor(x[3],y[2],z[2])")); builder.insertCell({0, 0, 0}, 10); builder.insertCell({0, 0, 1}, 11); builder.insertCell({0, 1, 0}, 12); @@ -99,16 +104,26 @@ TEST("require that 3d tensor can be constructed") { *builder.build()); } +TEST("require that 2d tensor with float cells can be constructed") { + BuilderFlt builder(ValueType::from_spec("tensor<float>(x[3],y[2])")); + builder.insertCell({0, 1}, 2.5); + builder.insertCell({1, 0}, 1.5); + builder.insertCell({2, 0}, -0.25); + builder.insertCell({2, 1}, 0.75); + assertTensor("tensor<float>(x[3],y[2])", {0,2.5,1.5,0,-0.25,0.75}, + *builder.build()); +} + TEST("require that cells get default value 0 if not specified") { - Builder builder(ValueType::from_spec("tensor(x[3])")); + BuilderDbl builder(ValueType::from_spec("tensor(x[3])")); builder.insertCell(1, 11); assertTensor("tensor(x[3])", {0,11,0}, *builder.build()); } -void assertTensorCell(const DenseTensor::Address &expAddress, +void assertTensorCell(const DenseTensorView::Address &expAddress, double expCell, - const DenseTensor::CellsIterator &itr) + const DenseTensorView::CellsIterator &itr) { EXPECT_TRUE(itr.valid()); EXPECT_EQUAL(expAddress, itr.address()); @@ -118,14 +133,14 @@ void assertTensorCell(const DenseTensor::Address &expAddress, TEST("require that dense tensor cells iterator works for 1d tensor") { Tensor::UP tensor; { - Builder builder(ValueType::from_spec("tensor(x[2])")); + BuilderDbl builder(ValueType::from_spec("tensor(x[2])")); builder.insertCell(0, 2); builder.insertCell(1, 3); tensor = builder.build(); } - const DenseTensor &denseTensor = dynamic_cast<const DenseTensor &>(*tensor); - DenseTensor::CellsIterator itr = denseTensor.cellsIterator(); + const DenseTensorView &denseTensor = dynamic_cast<const DenseTensorView &>(*tensor); + DenseTensorView::CellsIterator itr = denseTensor.cellsIterator(); assertTensorCell({0}, 2, itr); itr.next(); @@ -137,7 +152,7 @@ TEST("require that dense tensor cells iterator works for 1d tensor") { TEST("require that dense tensor cells iterator works for 2d tensor") { Tensor::UP tensor; { - Builder builder(ValueType::from_spec("tensor(x[2],y[2])")); + BuilderDbl builder(ValueType::from_spec("tensor(x[2],y[2])")); builder.insertCell({0, 0}, 2); builder.insertCell({0, 1}, 3); builder.insertCell({1, 0}, 5); @@ -145,8 +160,8 @@ TEST("require that dense tensor cells iterator works for 2d tensor") { tensor = builder.build(); } - const DenseTensor &denseTensor = dynamic_cast<const DenseTensor &>(*tensor); - DenseTensor::CellsIterator itr = denseTensor.cellsIterator(); + const DenseTensorView &denseTensor = dynamic_cast<const DenseTensorView &>(*tensor); + DenseTensorView::CellsIterator itr = denseTensor.cellsIterator(); assertTensorCell({0,0}, 2, itr); itr.next(); diff --git a/eval/src/tests/tensor/tensor_mapper/.gitignore b/eval/src/tests/tensor/tensor_mapper/.gitignore deleted file mode 100644 index 8a312ff3157..00000000000 --- a/eval/src/tests/tensor/tensor_mapper/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespalib_tensor_mapper_test_app diff --git a/eval/src/tests/tensor/tensor_mapper/CMakeLists.txt b/eval/src/tests/tensor/tensor_mapper/CMakeLists.txt deleted file mode 100644 index dfc35b4f018..00000000000 --- a/eval/src/tests/tensor/tensor_mapper/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_tensor_mapper_test_app TEST - SOURCES - tensor_mapper_test.cpp - DEPENDS - vespaeval -) -vespa_add_test(NAME eval_tensor_mapper_test_app COMMAND eval_tensor_mapper_test_app) diff --git a/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp b/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp deleted file mode 100644 index 60b17930f0b..00000000000 --- a/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/tensor/tensor_mapper.h> -#include <vespa/eval/tensor/test/test_utils.h> -#include <vespa/eval/tensor/wrapped_simple_tensor.h> -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/stringfmt.h> - -using vespalib::eval::ValueType; -using vespalib::eval::Value; -using vespalib::eval::TensorSpec; -using vespalib::eval::SimpleTensor; -using vespalib::tensor::test::makeTensor; -using namespace vespalib::tensor; - -void -verify_wrapped(const TensorSpec &source, const vespalib::string &type, const TensorSpec &expect) -{ - auto tensor = std::make_unique<WrappedSimpleTensor>(SimpleTensor::create(source)); - auto mapped = TensorMapper::mapToWrapped(*tensor, ValueType::from_spec(type)); - TensorSpec actual = mapped->toSpec(); - EXPECT_EQUAL(actual, expect); -} - -void -verify(const TensorSpec &source, const vespalib::string &type, const TensorSpec &expect) -{ - auto tensor = makeTensor<Tensor>(source); - TensorMapper mapper(ValueType::from_spec(type)); - auto mapped = mapper.map(*tensor); - TensorSpec actual = mapped->toSpec(); - EXPECT_EQUAL(actual, expect); - TEST_DO(verify_wrapped(source, type, expect)); -} - -TEST("require that sparse tensors can be mapped to sparse type") { - TEST_DO(verify(TensorSpec("tensor(x{},y{})") - .add({{"x","1"},{"y","1"}}, 1) - .add({{"x","2"},{"y","1"}}, 3) - .add({{"x","1"},{"y","2"}}, 5) - .add({{"x","2"},{"y","2"}}, 7), - "tensor(y{})", - TensorSpec("tensor(y{})") - .add({{"y","1"}}, 4) - .add({{"y","2"}}, 12))); - - TEST_DO(verify(TensorSpec("tensor(x{},y{})") - .add({{"x","1"},{"y","1"}}, 1) - .add({{"x","2"},{"y","1"}}, 3) - .add({{"x","1"},{"y","2"}}, 5) - .add({{"x","2"},{"y","2"}}, 7), - "tensor(x{})", - TensorSpec("tensor(x{})") - .add({{"x","1"}}, 6) - .add({{"x","2"}}, 10))); -} - -TEST("require that sparse tensors can be mapped to dense type") { - TEST_DO(verify(TensorSpec("tensor(x{},y{})") - .add({{"x","1"},{"y","0"}}, 1) - .add({{"x","2"},{"y","0"}}, 3) - .add({{"x","1"},{"y","1"}}, 5) - .add({{"x","2"},{"y","1"}}, 7), - "tensor(y[3])", - TensorSpec("tensor(y[3])") - .add({{"y",0}}, 4) - .add({{"y",1}}, 12) - .add({{"y",2}}, 0))); - - TEST_DO(verify(TensorSpec("tensor(x{},y{})") - .add({{"x","1"},{"y","0x"}}, 1) - .add({{"x","2"},{"y",""}}, 3) - .add({{"x","1"},{"y","1"}}, 5) - .add({{"x","2"},{"y","10"}}, 7), - "tensor(y[3])", - TensorSpec("tensor(y[3])") - .add({{"y",0}}, 3) - .add({{"y",1}}, 5) - .add({{"y",2}}, 0))); - - TEST_DO(verify(TensorSpec("tensor(x{},y{})") - .add({{"x","0"},{"y","0"}}, 1) - .add({{"x","1"},{"y","0"}}, 3) - .add({{"x","0"},{"y","1"}}, 5) - .add({{"x","10"},{"y","1"}}, 7), - "tensor(x[2],y[3])", - TensorSpec("tensor(x[2],y[3])") - .add({{"x",0},{"y",0}}, 1) - .add({{"x",0},{"y",1}}, 5) - .add({{"x",0},{"y",2}}, 0) - .add({{"x",1},{"y",0}}, 3) - .add({{"x",1},{"y",1}}, 0) - .add({{"x",1},{"y",2}}, 0))); -} - -TEST("require that dense tensors can be mapped to sparse type") { - TEST_DO(verify(TensorSpec("tensor(x[2],y[2])") - .add({{"x",0},{"y",0}}, 1) - .add({{"x",0},{"y",1}}, 3) - .add({{"x",1},{"y",0}}, 5) - .add({{"x",1},{"y",1}}, 7), - "tensor(x{})", - TensorSpec("tensor(x{})") - .add({{"x","0"}}, 4) - .add({{"x","1"}}, 12))); -} - -TEST("require that mixed tensors can be mapped to sparse type") { - TEST_DO(verify(TensorSpec("tensor(x[2],y{})") - .add({{"x",0},{"y","0"}}, 1) - .add({{"x",0},{"y","1"}}, 3) - .add({{"x",1},{"y","0"}}, 5) - .add({{"x",1},{"y","1"}}, 7), - "tensor(x{})", - TensorSpec("tensor(x{})") - .add({{"x","0"}}, 4) - .add({{"x","1"}}, 12))); -} - -TEST("require that mixed tensors can be mapped to dense type") { - TEST_DO(verify(TensorSpec("tensor(x[2],y{})") - .add({{"x",0},{"y","0"}}, 1) - .add({{"x",0},{"y","1"}}, 3) - .add({{"x",1},{"y","0"}}, 5) - .add({{"x",1},{"y","1"}}, 7), - "tensor(y[2])", - TensorSpec("tensor(y[2])") - .add({{"y",0}}, 6) - .add({{"y",1}}, 10))); -} - -TEST("require that mixed tensors can be mapped to mixed type") { - TEST_DO(verify(TensorSpec("tensor(x[2],y{})") - .add({{"x",0},{"y","0"}}, 1) - .add({{"x",0},{"y","1"}}, 3) - .add({{"x",1},{"y","0"}}, 5) - .add({{"x",1},{"y","1"}}, 7), - "tensor(x{},y[2])", - TensorSpec("tensor(x{},y[2])") - .add({{"x","0"},{"y",0}}, 1) - .add({{"x","0"},{"y",1}}, 3) - .add({{"x","1"},{"y",0}}, 5) - .add({{"x","1"},{"y",1}}, 7))); -} - -TEST("require that dense tensors can be mapped to mixed type") { - TEST_DO(verify(TensorSpec("tensor(x[2],y[2])") - .add({{"x",0},{"y",0}}, 1) - .add({{"x",0},{"y",1}}, 3) - .add({{"x",1},{"y",0}}, 5) - .add({{"x",1},{"y",1}}, 7), - "tensor(x{},y[2])", - TensorSpec("tensor(x{},y[2])") - .add({{"x","0"},{"y",0}}, 1) - .add({{"x","0"},{"y",1}}, 3) - .add({{"x","1"},{"y",0}}, 5) - .add({{"x","1"},{"y",1}}, 7))); -} - -TEST("require that sparse tensors can be mapped to mixed type") { - TEST_DO(verify(TensorSpec("tensor(x{},y{})") - .add({{"x","0"},{"y","0"}}, 1) - .add({{"x","0"},{"y","1"}}, 3) - .add({{"x","1"},{"y","0"}}, 5) - .add({{"x","1"},{"y","1"}}, 7), - "tensor(x[2],y{})", - TensorSpec("tensor(x[2],y{})") - .add({{"x",0},{"y","0"}}, 1) - .add({{"x",0},{"y","1"}}, 3) - .add({{"x",1},{"y","0"}}, 5) - .add({{"x",1},{"y","1"}}, 7))); -} - -TEST("require that missing dimensions are added appropriately") { - TEST_DO(verify(TensorSpec("tensor(x{})") - .add({{"x","foo"}}, 42), - "tensor(x{},y{})", - TensorSpec("tensor(x{},y{})") - .add({{"x","foo"},{"y",""}}, 42))); - - TEST_DO(verify(TensorSpec("tensor(x[1])") - .add({{"x",0}}, 42), - "tensor(x[1],y[1],z[2])", - TensorSpec("tensor(x[1],y[1],z[2])") - .add({{"x",0},{"y",0},{"z",0}}, 42) - .add({{"x",0},{"y",0},{"z",1}}, 0))); - - TEST_DO(verify(TensorSpec("tensor(a{})") - .add({{"a","foo"}}, 42), - "tensor(a{},b[1],c{},d[2])", - TensorSpec("tensor(a{},b[1],c{},d[2])") - .add({{"a","foo"},{"b",0},{"c",""},{"d",0}}, 42) - .add({{"a","foo"},{"b",0},{"c",""},{"d",1}}, 0))); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/eval/tensor_spec.h b/eval/src/vespa/eval/eval/tensor_spec.h index 32dc1c82fcb..25af4c7a93c 100644 --- a/eval/src/vespa/eval/eval/tensor_spec.h +++ b/eval/src/vespa/eval/eval/tensor_spec.h @@ -73,7 +73,6 @@ public: } return *this; } - void override_type(const vespalib::string &new_type) { _type = new_type; } const vespalib::string &type() const { return _type; } const Cells &cells() const { return _cells; } vespalib::string to_string() const; diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp index 321b472a3fa..3f5fa4d72bb 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp +++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp @@ -14,7 +14,7 @@ NodeTypes get_types(const Function &function, const ParamRepo ¶m_repo) { for (size_t i = 0; i < function.num_params(); ++i) { auto pos = param_repo.map.find(function.param_name(i)); ASSERT_TRUE(pos != param_repo.map.end()); - param_types.push_back(ValueType::from_spec(pos->second.type)); + param_types.push_back(ValueType::from_spec(pos->second.value.type())); ASSERT_TRUE(!param_types.back().is_error()); } return NodeTypes(function, param_types); diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h index 8c7d15e7416..9c793c01861 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.h +++ b/eval/src/vespa/eval/eval/test/eval_fixture.h @@ -18,32 +18,24 @@ class EvalFixture public: struct Param { TensorSpec value; // actual parameter value - vespalib::string type; // pre-defined type (could be abstract) bool is_mutable; // input will be mutable (if allow_mutable is true) - Param(TensorSpec value_in, const vespalib::string &type_in, bool is_mutable_in) - : value(std::move(value_in)), type(type_in), is_mutable(is_mutable_in) {} + Param(TensorSpec value_in, bool is_mutable_in) + : value(std::move(value_in)), is_mutable(is_mutable_in) {} ~Param() {} }; struct ParamRepo { std::map<vespalib::string,Param> map; ParamRepo() : map() {} - ParamRepo &add(const vespalib::string &name, TensorSpec value_in, const vespalib::string &type_in, bool is_mutable_in) { - value_in.override_type(type_in); - map.insert_or_assign(name, Param(std::move(value_in), type_in, is_mutable_in)); + ParamRepo &add(const vespalib::string &name, TensorSpec value_in, bool is_mutable_in) { + map.insert_or_assign(name, Param(std::move(value_in), is_mutable_in)); return *this; } - ParamRepo &add(const vespalib::string &name, TensorSpec value, const vespalib::string &type) { - return add(name, value, type, false); - } - ParamRepo &add_mutable(const vespalib::string &name, TensorSpec value, const vespalib::string &type) { - return add(name, value, type, true); - } ParamRepo &add(const vespalib::string &name, const TensorSpec &value) { - return add(name, value, value.type(), false); + return add(name, value, false); } ParamRepo &add_mutable(const vespalib::string &name, const TensorSpec &value) { - return add(name, value, value.type(), true); + return add(name, value, true); } ~ParamRepo() {} }; diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp index 1e1bd828d41..dc39dfb04a7 100644 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp +++ b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp @@ -301,10 +301,13 @@ struct TestContext { TEST_DO(verify_create_type("double")); TEST_DO(verify_create_type("tensor(x{})")); TEST_DO(verify_create_type("tensor(x{},y{})")); + TEST_DO(verify_create_type("tensor<float>(x{},y{})")); TEST_DO(verify_create_type("tensor(x[5])")); TEST_DO(verify_create_type("tensor(x[5],y[10])")); + TEST_DO(verify_create_type("tensor<float>(x[5],y[10])")); TEST_DO(verify_create_type("tensor(x{},y[10])")); - TEST_DO(verify_create_type("tensor(x[5],y{})")); + TEST_DO(verify_create_type("tensor(x[5],y{})")); + TEST_DO(verify_create_type("tensor<float>(x[5],y{})")); } //------------------------------------------------------------------------- @@ -318,11 +321,14 @@ struct TestContext { {x(3)}, {x(3),y(5)}, {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar"})}, {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})} + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) }; for (const Layout &layout: layouts) { TensorSpec input = spec(layout, seq); @@ -349,7 +355,7 @@ struct TestContext { void test_tensor_reduce() { TEST_DO(test_reduce_op(Aggr::AVG, N())); TEST_DO(test_reduce_op(Aggr::COUNT, N())); - TEST_DO(test_reduce_op(Aggr::PROD, Sigmoid(N()))); + TEST_DO(test_reduce_op(Aggr::PROD, SigmoidF(N()))); TEST_DO(test_reduce_op(Aggr::SUM, N())); TEST_DO(test_reduce_op(Aggr::MAX, N())); TEST_DO(test_reduce_op(Aggr::MIN, N())); @@ -363,11 +369,14 @@ struct TestContext { {x(3)}, {x(3),y(5)}, {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar"})}, {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})} + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) }; for (const Layout &layout: layouts) { TEST_DO(verify_result(eval.eval(engine, spec(layout, seq)), spec(layout, OpSeq(seq, ref_op)))); @@ -381,30 +390,30 @@ struct TestContext { } void test_tensor_map() { - TEST_DO(test_map_op("-a", operation::Neg::f, Sub2(Div10(N())))); + TEST_DO(test_map_op("-a", operation::Neg::f, Sub2(Div16(N())))); TEST_DO(test_map_op("!a", operation::Not::f, Mask2Seq(SkipNth(3)))); - TEST_DO(test_map_op("cos(a)", operation::Cos::f, Div10(N()))); - TEST_DO(test_map_op("sin(a)", operation::Sin::f, Div10(N()))); - TEST_DO(test_map_op("tan(a)", operation::Tan::f, Div10(N()))); - TEST_DO(test_map_op("cosh(a)", operation::Cosh::f, Div10(N()))); - TEST_DO(test_map_op("sinh(a)", operation::Sinh::f, Div10(N()))); - TEST_DO(test_map_op("tanh(a)", operation::Tanh::f, Div10(N()))); - TEST_DO(test_map_op("acos(a)", operation::Acos::f, Sigmoid(Div10(N())))); - TEST_DO(test_map_op("asin(a)", operation::Asin::f, Sigmoid(Div10(N())))); - TEST_DO(test_map_op("atan(a)", operation::Atan::f, Div10(N()))); - TEST_DO(test_map_op("exp(a)", operation::Exp::f, Div10(N()))); - TEST_DO(test_map_op("log10(a)", operation::Log10::f, Div10(N()))); - TEST_DO(test_map_op("log(a)", operation::Log::f, Div10(N()))); - TEST_DO(test_map_op("sqrt(a)", operation::Sqrt::f, Div10(N()))); - TEST_DO(test_map_op("ceil(a)", operation::Ceil::f, Div10(N()))); - TEST_DO(test_map_op("fabs(a)", operation::Fabs::f, Div10(N()))); - TEST_DO(test_map_op("floor(a)", operation::Floor::f, Div10(N()))); + TEST_DO(test_map_op("cos(a)", operation::Cos::f, Div16(N()))); + TEST_DO(test_map_op("sin(a)", operation::Sin::f, Div16(N()))); + TEST_DO(test_map_op("tan(a)", operation::Tan::f, Div16(N()))); + TEST_DO(test_map_op("cosh(a)", operation::Cosh::f, Div16(N()))); + TEST_DO(test_map_op("sinh(a)", operation::Sinh::f, Div16(N()))); + TEST_DO(test_map_op("tanh(a)", operation::Tanh::f, Div16(N()))); + TEST_DO(test_map_op("acos(a)", operation::Acos::f, SigmoidF(Div16(N())))); + TEST_DO(test_map_op("asin(a)", operation::Asin::f, SigmoidF(Div16(N())))); + TEST_DO(test_map_op("atan(a)", operation::Atan::f, Div16(N()))); + TEST_DO(test_map_op("exp(a)", operation::Exp::f, Div16(N()))); + TEST_DO(test_map_op("log10(a)", operation::Log10::f, Div16(N()))); + TEST_DO(test_map_op("log(a)", operation::Log::f, Div16(N()))); + TEST_DO(test_map_op("sqrt(a)", operation::Sqrt::f, Div16(N()))); + TEST_DO(test_map_op("ceil(a)", operation::Ceil::f, Div16(N()))); + TEST_DO(test_map_op("fabs(a)", operation::Fabs::f, Div16(N()))); + TEST_DO(test_map_op("floor(a)", operation::Floor::f, Div16(N()))); TEST_DO(test_map_op("isNan(a)", operation::IsNan::f, Mask2Seq(SkipNth(3), 1.0, my_nan))); - TEST_DO(test_map_op("relu(a)", operation::Relu::f, Sub2(Div10(N())))); - TEST_DO(test_map_op("sigmoid(a)", operation::Sigmoid::f, Sub2(Div10(N())))); - TEST_DO(test_map_op("elu(a)", operation::Elu::f, Sub2(Div10(N())))); + TEST_DO(test_map_op("relu(a)", operation::Relu::f, Sub2(Div16(N())))); + TEST_DO(test_map_op("sigmoid(a)", operation::Sigmoid::f, Sub2(Div16(N())))); + TEST_DO(test_map_op("elu(a)", operation::Elu::f, Sub2(Div16(N())))); TEST_DO(test_map_op("a in [1,5,7,13,42]", MyIn::f, N())); - TEST_DO(test_map_op("(a+1)*2", MyOp::f, Div10(N()))); + TEST_DO(test_map_op("(a+1)*2", MyOp::f, Div16(N()))); } //------------------------------------------------------------------------- @@ -612,20 +621,29 @@ struct TestContext { void test_apply_op(const Eval &eval, join_fun_t op, const Sequence &seq) { std::vector<Layout> layouts = { - {}, {}, - {x(5)}, {x(5)}, - {x(5)}, {y(5)}, - {x(5)}, {x(5),y(5)}, - {y(3)}, {x(2),z(3)}, - {x(3),y(5)}, {y(5),z(7)}, - {x({"a","b","c"})}, {x({"a","b","c"})}, - {x({"a","b","c"})}, {x({"a","b"})}, - {x({"a","b","c"})}, {y({"foo","bar","baz"})}, - {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, - {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})} + {}, {}, + {x(5)}, {x(5)}, + {x(5)}, {y(5)}, + {x(5)}, {x(5),y(5)}, + {y(3)}, {x(2),z(3)}, + {x(3),y(5)}, {y(5),z(7)}, + float_cells({x(3),y(5)}), {y(5),z(7)}, + {x(3),y(5)}, float_cells({y(5),z(7)}), + float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), + {x({"a","b","c"})}, {x({"a","b","c"})}, + {x({"a","b","c"})}, {x({"a","b"})}, + {x({"a","b","c"})}, {y({"foo","bar","baz"})}, + {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, + {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), + float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) }; ASSERT_TRUE((layouts.size() % 2) == 0); for (size_t i = 0; i < layouts.size(); i += 2) { @@ -648,27 +666,27 @@ struct TestContext { } void test_tensor_apply() { - TEST_DO(test_apply_op("a+b", operation::Add::f, Div10(N()))); - TEST_DO(test_apply_op("a-b", operation::Sub::f, Div10(N()))); - TEST_DO(test_apply_op("a*b", operation::Mul::f, Div10(N()))); - TEST_DO(test_apply_op("a/b", operation::Div::f, Div10(N()))); - TEST_DO(test_apply_op("a%b", operation::Mod::f, Div10(N()))); - TEST_DO(test_apply_op("a^b", operation::Pow::f, Div10(N()))); - TEST_DO(test_apply_op("pow(a,b)", operation::Pow::f, Div10(N()))); - TEST_DO(test_apply_op("a==b", operation::Equal::f, Div10(N()))); - TEST_DO(test_apply_op("a!=b", operation::NotEqual::f, Div10(N()))); - TEST_DO(test_apply_op("a~=b", operation::Approx::f, Div10(N()))); - TEST_DO(test_apply_op("a<b", operation::Less::f, Div10(N()))); - TEST_DO(test_apply_op("a<=b", operation::LessEqual::f, Div10(N()))); - TEST_DO(test_apply_op("a>b", operation::Greater::f, Div10(N()))); - TEST_DO(test_apply_op("a>=b", operation::GreaterEqual::f, Div10(N()))); + TEST_DO(test_apply_op("a+b", operation::Add::f, Div16(N()))); + TEST_DO(test_apply_op("a-b", operation::Sub::f, Div16(N()))); + TEST_DO(test_apply_op("a*b", operation::Mul::f, Div16(N()))); + TEST_DO(test_apply_op("a/b", operation::Div::f, Div16(N()))); + TEST_DO(test_apply_op("a%b", operation::Mod::f, Div16(N()))); + TEST_DO(test_apply_op("a^b", operation::Pow::f, Div16(N()))); + TEST_DO(test_apply_op("pow(a,b)", operation::Pow::f, Div16(N()))); + TEST_DO(test_apply_op("a==b", operation::Equal::f, Div16(N()))); + TEST_DO(test_apply_op("a!=b", operation::NotEqual::f, Div16(N()))); + TEST_DO(test_apply_op("a~=b", operation::Approx::f, Div16(N()))); + TEST_DO(test_apply_op("a<b", operation::Less::f, Div16(N()))); + TEST_DO(test_apply_op("a<=b", operation::LessEqual::f, Div16(N()))); + TEST_DO(test_apply_op("a>b", operation::Greater::f, Div16(N()))); + TEST_DO(test_apply_op("a>=b", operation::GreaterEqual::f, Div16(N()))); TEST_DO(test_apply_op("a&&b", operation::And::f, Mask2Seq(SkipNth(3)))); TEST_DO(test_apply_op("a||b", operation::Or::f, Mask2Seq(SkipNth(3)))); - TEST_DO(test_apply_op("atan2(a,b)", operation::Atan2::f, Div10(N()))); - TEST_DO(test_apply_op("ldexp(a,b)", operation::Ldexp::f, Div10(N()))); - TEST_DO(test_apply_op("fmod(a,b)", operation::Mod::f, Div10(N()))); - TEST_DO(test_apply_op("min(a,b)", operation::Min::f, Div10(N()))); - TEST_DO(test_apply_op("max(a,b)", operation::Max::f, Div10(N()))); + TEST_DO(test_apply_op("atan2(a,b)", operation::Atan2::f, Div16(N()))); + TEST_DO(test_apply_op("ldexp(a,b)", operation::Ldexp::f, Div16(N()))); + TEST_DO(test_apply_op("fmod(a,b)", operation::Mod::f, Div16(N()))); + TEST_DO(test_apply_op("min(a,b)", operation::Min::f, Div16(N()))); + TEST_DO(test_apply_op("max(a,b)", operation::Max::f, Div16(N()))); } //------------------------------------------------------------------------- @@ -681,10 +699,20 @@ struct TestContext { TEST_DO(verify_result(safe(eval).eval(engine, lhs, rhs), spec(expect))); } + void test_dot_product(double expect, + const Layout &lhs, const Seq &lhs_seq, + const Layout &rhs, const Seq &rhs_seq) + { + TEST_DO(test_dot_product(expect, spec(lhs, lhs_seq), spec(rhs, rhs_seq))); + TEST_DO(test_dot_product(expect, spec(float_cells(lhs), lhs_seq), spec(rhs, rhs_seq))); + TEST_DO(test_dot_product(expect, spec(lhs, lhs_seq), spec(float_cells(rhs), rhs_seq))); + TEST_DO(test_dot_product(expect, spec(float_cells(lhs), lhs_seq), spec(float_cells(rhs), rhs_seq))); + } + void test_dot_product() { TEST_DO(test_dot_product(((2 * 7) + (3 * 11) + (5 * 13)), - spec(x(3), Seq({ 2, 3, 5 })), - spec(x(3), Seq({ 7, 11, 13 })))); + {x(3)}, Seq({ 2, 3, 5 }), + {x(3)}, Seq({ 7, 11, 13 }))); } //------------------------------------------------------------------------- @@ -714,6 +742,16 @@ struct TestContext { spec({x(2),y(2),z(3)}, Seq({1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0})))); TEST_DO(test_concat(spec(y(2), Seq({1.0, 2.0})), spec(y(2), Seq({4.0, 5.0})), "x", spec({x(2), y(2)}, Seq({1.0, 2.0, 4.0, 5.0})))); + + TEST_DO(test_concat(spec(float_cells({x(1)}), Seq({10.0})), spec(20.0), "x", spec(float_cells({x(2)}), Seq({10.0, 20.0})))); + TEST_DO(test_concat(spec(10.0), spec(float_cells({x(1)}), Seq({20.0})), "x", spec(float_cells({x(2)}), Seq({10.0, 20.0})))); + + TEST_DO(test_concat(spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})), spec(x(2), Seq({4.0, 5.0})), "x", + spec(x(5), Seq({1.0, 2.0, 3.0, 4.0, 5.0})))); + TEST_DO(test_concat(spec(x(3), Seq({1.0, 2.0, 3.0})), spec(float_cells({x(2)}), Seq({4.0, 5.0})), "x", + spec(x(5), Seq({1.0, 2.0, 3.0, 4.0, 5.0})))); + TEST_DO(test_concat(spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})), spec(float_cells({x(2)}), Seq({4.0, 5.0})), "x", + spec(float_cells({x(5)}), Seq({1.0, 2.0, 3.0, 4.0, 5.0})))); } //------------------------------------------------------------------------- @@ -732,6 +770,7 @@ struct TestContext { void test_rename() { TEST_DO(test_rename("rename(a,x,y)", spec(x(5), N()), {"x"}, {"y"}, spec(y(5), N()))); TEST_DO(test_rename("rename(a,y,x)", spec({y(5),z(5)}, N()), {"y"}, {"x"}, spec({x(5),z(5)}, N()))); + TEST_DO(test_rename("rename(a,y,x)", spec(float_cells({y(5),z(5)}), N()), {"y"}, {"x"}, spec(float_cells({x(5),z(5)}), N()))); TEST_DO(test_rename("rename(a,z,x)", spec({y(5),z(5)}, N()), {"z"}, {"x"}, spec({y(5),x(5)}, N()))); TEST_DO(test_rename("rename(a,x,z)", spec({x(5),y(5)}, N()), {"x"}, {"z"}, spec({z(5),y(5)}, N()))); TEST_DO(test_rename("rename(a,y,z)", spec({x(5),y(5)}, N()), {"y"}, {"z"}, spec({x(5),z(5)}, N()))); @@ -746,6 +785,7 @@ struct TestContext { void test_tensor_lambda() { TEST_DO(test_tensor_lambda("tensor(x[10])(x+1)", spec(x(10), N()))); + TEST_DO(test_tensor_lambda("tensor<float>(x[10])(x+1)", spec(float_cells({x(10)}), N()))); TEST_DO(test_tensor_lambda("tensor(x[5],y[4])(x*4+(y+1))", spec({x(5),y(4)}, N()))); TEST_DO(test_tensor_lambda("tensor(x[5],y[4])(x==y)", spec({x(5),y(4)}, Seq({ 1.0, 0.0, 0.0, 0.0, @@ -818,11 +858,14 @@ struct TestContext { TEST_DO(verify_encode_decode(spec({x(3)}, N()))); TEST_DO(verify_encode_decode(spec({x(3),y(5)}, N()))); TEST_DO(verify_encode_decode(spec({x(3),y(5),z(7)}, N()))); + TEST_DO(verify_encode_decode(spec(float_cells({x(3),y(5),z(7)}), N()))); TEST_DO(verify_encode_decode(spec({x({"a","b","c"})}, N()))); TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y({"foo","bar"})}, N()))); TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, N()))); + TEST_DO(verify_encode_decode(spec(float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), N()))); TEST_DO(verify_encode_decode(spec({x(3),y({"foo", "bar"}),z(7)}, N()))); TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y(5),z({"i","j","k","l"})}, N()))); + TEST_DO(verify_encode_decode(spec(float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}), N()))); } //------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/test/tensor_model.hpp b/eval/src/vespa/eval/eval/test/tensor_model.hpp index 50a7b6a639a..6efb7470d55 100644 --- a/eval/src/vespa/eval/eval/test/tensor_model.hpp +++ b/eval/src/vespa/eval/eval/test/tensor_model.hpp @@ -10,6 +10,7 @@ namespace vespalib { namespace eval { namespace test { +using CellType = ValueType::CellType; using map_fun_t = TensorEngine::map_fun_t; using join_fun_t = TensorEngine::join_fun_t; @@ -31,6 +32,14 @@ struct Div10 : Sequence { double operator[](size_t i) const override { return (seq[i] / 10.0); } }; +// Sequence of another sequence divided by 10 +struct Div16 : Sequence { + const Sequence &seq; + Div16(const Sequence &seq_in) : seq(seq_in) {} + double operator[](size_t i) const override { return (seq[i] / 16.0); } +}; + + // Sequence of another sequence minus 2 struct Sub2 : Sequence { const Sequence &seq; @@ -53,6 +62,13 @@ struct Sigmoid : Sequence { double operator[](size_t i) const override { return operation::Sigmoid::f(seq[i]); } }; +// Sequence of applying sigmoid to another sequence, plus rounding to nearest float +struct SigmoidF : Sequence { + const Sequence &seq; + SigmoidF(const Sequence &seq_in) : seq(seq_in) {} + double operator[](size_t i) const override { return (float)operation::Sigmoid::f(seq[i]); } +}; + // pre-defined sequence of numbers struct Seq : Sequence { std::vector<double> seq; @@ -146,7 +162,22 @@ struct Domain { Domain::Domain(const Domain &) = default; Domain::~Domain() {} -using Layout = std::vector<Domain>; +struct Layout { + CellType cell_type; + std::vector<Domain> domains; + Layout(std::initializer_list<Domain> domains_in) + : cell_type(CellType::DOUBLE), domains(domains_in) {} + Layout(CellType cell_type_in, std::vector<Domain> domains_in) + : cell_type(cell_type_in), domains(std::move(domains_in)) {} + auto begin() const { return domains.begin(); } + auto end() const { return domains.end(); } + auto size() const { return domains.size(); } + auto operator[](size_t idx) const { return domains[idx]; } +}; + +Layout float_cells(const Layout &layout) { + return Layout(CellType::FLOAT, layout.domains); +} Domain x() { return Domain("x", {}); } Domain x(size_t size) { return Domain("x", size); } @@ -162,9 +193,6 @@ Domain z(const std::vector<vespalib::string> &keys) { return Domain("z", keys); // Infer the tensor type spanned by the given spaces vespalib::string infer_type(const Layout &layout) { - if (layout.empty()) { - return "double"; - } std::vector<ValueType::Dimension> dimensions; for (const auto &domain: layout) { if (domain.size == 0) { @@ -173,7 +201,7 @@ vespalib::string infer_type(const Layout &layout) { dimensions.emplace_back(domain.dimension, domain.size); // indexed } } - return ValueType::tensor_type(dimensions).to_spec(); + return ValueType::tensor_type(dimensions, layout.cell_type).to_spec(); } // Wrapper for the things needed to generate a tensor diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index 81788c933d7..423c8e15d92 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -16,7 +16,7 @@ class ValueType { public: enum class Type { ERROR, DOUBLE, TENSOR }; - enum class CellType { FLOAT, DOUBLE }; + enum class CellType : char { FLOAT, DOUBLE }; struct Dimension { using size_type = uint32_t; static constexpr size_type npos = -1; @@ -85,4 +85,9 @@ public: std::ostream &operator<<(std::ostream &os, const ValueType &type); -} +// utility template +template <typename T> inline bool check_cell_type(ValueType::CellType type); +template <> inline bool check_cell_type<double>(ValueType::CellType type) { return (type == ValueType::CellType::DOUBLE); } +template <> inline bool check_cell_type<float>(ValueType::CellType type) { return (type == ValueType::CellType::FLOAT); } + +} // namespace diff --git a/eval/src/vespa/eval/tensor/CMakeLists.txt b/eval/src/vespa/eval/tensor/CMakeLists.txt index 4e9940d3c0a..bc0a4d340b8 100644 --- a/eval/src/vespa/eval/tensor/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/CMakeLists.txt @@ -5,6 +5,5 @@ vespa_add_library(eval_tensor OBJECT tensor.cpp tensor_address.cpp tensor_apply.cpp - tensor_mapper.cpp wrapped_simple_tensor.cpp ) diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index dc658d0b2da..58db90f5557 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -7,7 +7,7 @@ #include "sparse/sparse_tensor_address_builder.h" #include "sparse/direct_sparse_tensor_builder.h" #include "dense/dense_tensor.h" -#include "dense/direct_dense_tensor_builder.h" +#include "dense/typed_dense_tensor_builder.h" #include "dense/dense_dot_product_function.h" #include "dense/dense_xw_product_function.h" #include "dense/dense_fast_rename_optimizer.h" @@ -159,6 +159,24 @@ DefaultTensorEngine::to_spec(const Value &value) const } } +struct CallDenseTensorBuilder { + template <typename CT> + static Value::UP + call(const ValueType &type, const TensorSpec &spec) + { + TypedDenseTensorBuilder<CT> builder(type); + for (const auto &cell: spec.cells()) { + const auto &address = cell.first; + size_t cell_idx = calculate_cell_index(type, address); + if (cell_idx == UNDEFINED_IDX) { + bad_spec(spec); + } + builder.insertCell(cell_idx, cell.second); + } + return builder.build(); + } +}; + Value::UP DefaultTensorEngine::from_spec(const TensorSpec &spec) const { @@ -169,16 +187,7 @@ DefaultTensorEngine::from_spec(const TensorSpec &spec) const double value = spec.cells().empty() ? 0.0 : spec.cells().begin()->second.value; return std::make_unique<DoubleValue>(value); } else if (type.is_dense()) { - DirectDenseTensorBuilder builder(type); - for (const auto &cell: spec.cells()) { - const auto &address = cell.first; - size_t cell_idx = calculate_cell_index(type, address); - if (cell_idx == UNDEFINED_IDX) { - bad_spec(spec); - } - builder.insertCell(cell_idx, cell.second); - } - return builder.build(); + return dispatch_0<CallDenseTensorBuilder>(type.cell_type(), type, spec); } else if (type.is_sparse()) { DirectSparseTensorBuilder builder(type); SparseTensorAddressBuilder address_builder; @@ -223,7 +232,7 @@ DefaultTensorEngine::encode(const Value &value, nbostream &output) const if (auto tensor = value.as_tensor()) { TypedBinaryFormat::serialize(output, static_cast<const tensor::Tensor &>(*tensor)); } else { - TypedBinaryFormat::serialize(output, DenseTensor(ValueType::double_type(), {value.as_double()})); + TypedBinaryFormat::serialize(output, DenseTensor<double>(ValueType::double_type(), {value.as_double()})); } } @@ -357,12 +366,18 @@ size_t vector_size(const ValueType &type, const vespalib::string &dimension) { } } +struct CallAppendVector { + template <typename CT> + static void call(const ConstArrayRef<CT> &arr, double *&pos) { + for (CT cell : arr) { *pos++ = cell; } + } +}; + void append_vector(double *&pos, const Value &value) { if (auto tensor = value.as_tensor()) { const DenseTensorView *view = static_cast<const DenseTensorView *>(tensor); - for (double cell: view->cellsRef()) { - *pos++ = cell; - } + TypedCells cellsRef = view->cellsRef(); + dispatch_1<CallAppendVector>(cellsRef, pos); } else { *pos++ = value.as_double(); } @@ -375,7 +390,7 @@ const Value &concat_vectors(const Value &a, const Value &b, const vespalib::stri append_vector(pos, b); assert(pos == cells.end()); const ValueType &type = stash.create<ValueType>(ValueType::tensor_type({ValueType::Dimension(dimension, vector_size)})); - return stash.create<DenseTensorView>(type, cells); + return stash.create<DenseTensorView>(type, TypedCells(cells)); } const Value & diff --git a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt index ce20d6ba6d9..78723fddc17 100644 --- a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt @@ -16,7 +16,8 @@ vespa_add_library(eval_tensor_dense OBJECT dense_tensor_reduce.cpp dense_tensor_view.cpp dense_xw_product_function.cpp - direct_dense_tensor_builder.cpp mutable_dense_tensor_view.cpp + typed_cells.cpp + typed_dense_tensor_builder.cpp vector_from_doubles_function.cpp ) diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp index 988edba7d55..c925f288c4a 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp @@ -9,7 +9,6 @@ namespace vespalib::tensor { -using CellsRef = DenseTensorView::CellsRef; using eval::ValueType; using eval::TensorFunction; using eval::as; @@ -19,17 +18,19 @@ using namespace eval::operation; namespace { -CellsRef getCellsRef(const eval::Value &value) { +TypedCells getCellsRef(const eval::Value &value) { const DenseTensorView &denseTensor = static_cast<const DenseTensorView &>(value); return denseTensor.cellsRef(); } void my_dot_product_op(eval::InterpretedFunction::State &state, uint64_t param) { auto *hw_accelerator = (hwaccelrated::IAccelrated *)(param); - DenseTensorView::CellsRef lhsCells = getCellsRef(state.peek(1)); - DenseTensorView::CellsRef rhsCells = getCellsRef(state.peek(0)); - size_t numCells = std::min(lhsCells.size(), rhsCells.size()); - double result = hw_accelerator->dotProduct(lhsCells.cbegin(), rhsCells.cbegin(), numCells); + TypedCells lhsCells = getCellsRef(state.peek(1)); + TypedCells rhsCells = getCellsRef(state.peek(0)); + size_t numCells = std::min(lhsCells.size, rhsCells.size); + const ConstArrayRef<double> lhs = lhsCells.typify<double>(); + const ConstArrayRef<double> rhs = rhsCells.typify<double>(); + double result = hw_accelerator->dotProduct(lhs.cbegin(), rhs.cbegin(), numCells); state.pop_pop_push(state.stash.create<eval::DoubleValue>(result)); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_fast_rename_optimizer.cpp b/eval/src/vespa/eval/tensor/dense/dense_fast_rename_optimizer.cpp index 09977df25b7..d8e1876ac64 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_fast_rename_optimizer.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_fast_rename_optimizer.cpp @@ -9,7 +9,6 @@ namespace vespalib::tensor { -using CellsRef = DenseTensorView::CellsRef; using eval::Value; using eval::ValueType; using eval::TensorFunction; diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.h b/eval/src/vespa/eval/tensor/dense/dense_generic_join.h index 49e075f6999..daf678d4916 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.h +++ b/eval/src/vespa/eval/tensor/dense/dense_generic_join.h @@ -4,7 +4,6 @@ namespace vespalib::tensor { class Tensor; - class DenseTensor; } namespace vespalib::tensor::dense { @@ -16,9 +15,10 @@ namespace vespalib::tensor::dense { */ template <typename Function> std::unique_ptr<Tensor> -apply(const DenseTensorView &lhs, const Tensor &rhs, Function &&func); +generic_join(const DenseTensorView &lhs, const Tensor &rhs, Function &&func); + template <typename Function> std::unique_ptr<Tensor> -apply(const DenseTensorView &lhs, const DenseTensorView &rhs, Function &&func); +generic_join(const DenseTensorView &lhs, const DenseTensorView &rhs, Function &&func); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_generic_join.hpp b/eval/src/vespa/eval/tensor/dense/dense_generic_join.hpp new file mode 100644 index 00000000000..aa08e6982bb --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_generic_join.hpp @@ -0,0 +1,66 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "dense_generic_join.h" +#include "dense_dimension_combiner.h" +#include "typed_dense_tensor_builder.h" + +namespace vespalib::tensor::dense { + +template <typename LCT, typename RCT, typename OCT, typename Function> +std::unique_ptr<Tensor> +generic_join(DenseDimensionCombiner & combiner, + TypedDenseTensorBuilder<OCT> & builder, + const ConstArrayRef<LCT> & lhsCells, + const ConstArrayRef<RCT> & rhsCells, Function &&func) __attribute__((noinline)); + +template <typename LCT, typename RCT, typename OCT, typename Function> +std::unique_ptr<Tensor> +generic_join(DenseDimensionCombiner & combiner, + TypedDenseTensorBuilder<OCT> & builder, + const ConstArrayRef<LCT> & lhsCells, + const ConstArrayRef<RCT> & rhsCells, Function &&func) +{ + for (combiner.leftReset(); combiner.leftInRange(); combiner.stepLeft()) { + for (combiner.rightReset(); combiner.rightInRange(); combiner.stepRight()) { + for (combiner.commonReset(); combiner.commonInRange(); combiner.stepCommon()) { + size_t outIdx = combiner.outputIdx(); + size_t l = combiner.leftIdx(); + size_t r = combiner.rightIdx(); + builder.insertCell(outIdx, func(lhsCells[l], rhsCells[r])); + } + } + } + return builder.build(); +} + +struct CallGenericJoin { + template <typename LCT, typename RCT, typename Function> + static std::unique_ptr<Tensor> + call(const ConstArrayRef<LCT> & lhsArr, + const ConstArrayRef<RCT> & rhsArr, + DenseDimensionCombiner & combiner, + Function &&func) + { + using OCT = typename OutputCellType<LCT, RCT>::output_type; + TypedDenseTensorBuilder<OCT> builder(combiner.result_type); + return generic_join(combiner, builder, lhsArr, rhsArr, std::move(func)); + } +}; + +template <typename Function> +std::unique_ptr<Tensor> +generic_join(const DenseTensorView &lhs, const Tensor &rhs, Function &&func) +{ + const DenseTensorView *view = dynamic_cast<const DenseTensorView *>(&rhs); + if (view) { + DenseDimensionCombiner combiner(lhs.fast_type(), view->fast_type()); + TypedCells lhsCells = lhs.cellsRef(); + TypedCells rhsCells = view->cellsRef(); + return dispatch_2<CallGenericJoin>(lhsCells, rhsCells, combiner, std::move(func)); + } + return Tensor::UP(); +} + +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.cpp index ce6b1743951..5fdfdbc4e9f 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.cpp @@ -9,7 +9,6 @@ namespace vespalib::tensor { -using CellsRef = DenseTensorView::CellsRef; using eval::Value; using eval::ValueType; using eval::TensorFunction; @@ -18,7 +17,7 @@ using namespace eval::tensor_function; namespace { -CellsRef getCellsRef(const eval::Value &value) { +TypedCells getCellsRef(const eval::Value &value) { const DenseTensorView &denseTensor = static_cast<const DenseTensorView &>(value); return denseTensor.cellsRef(); } @@ -26,8 +25,8 @@ CellsRef getCellsRef(const eval::Value &value) { template <bool write_left> void my_inplace_join_op(eval::InterpretedFunction::State &state, uint64_t param) { join_fun_t function = (join_fun_t)param; - CellsRef lhs_cells = getCellsRef(state.peek(1)); - CellsRef rhs_cells = getCellsRef(state.peek(0)); + ConstArrayRef<double> lhs_cells = getCellsRef(state.peek(1)).typify<double>(); + ConstArrayRef<double> rhs_cells = getCellsRef(state.peek(0)).typify<double>(); auto dst_cells = unconstify(write_left ? lhs_cells : rhs_cells); for (size_t i = 0; i < dst_cells.size(); ++i) { dst_cells[i] = function(lhs_cells[i], rhs_cells[i]); diff --git a/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.cpp index c72889ca0ed..b38a6b175dc 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.cpp @@ -8,7 +8,6 @@ namespace vespalib::tensor { -using CellsRef = DenseTensorView::CellsRef; using eval::Value; using eval::ValueType; using eval::TensorFunction; @@ -19,7 +18,7 @@ namespace { ArrayRef<double> getMutableCells(const eval::Value &value) { const DenseTensorView &denseTensor = static_cast<const DenseTensorView &>(value); - return unconstify(denseTensor.cellsRef()); + return unconstify(denseTensor.cellsRef().typify<double>()); } void my_inplace_map_op(eval::InterpretedFunction::State &state, uint64_t param) { diff --git a/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp index 9deac5437e0..b81b0f2c876 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp @@ -6,7 +6,6 @@ namespace vespalib::tensor { -using CellsRef = DenseTensorView::CellsRef; using eval::Value; using eval::ValueType; using eval::TensorFunction; @@ -15,14 +14,14 @@ using namespace eval::tensor_function; namespace { -CellsRef getCellsRef(const eval::Value &value) { +TypedCells getCellsRef(const eval::Value &value) { const DenseTensorView &denseTensor = static_cast<const DenseTensorView &>(value); return denseTensor.cellsRef(); } void my_replace_type_op(eval::InterpretedFunction::State &state, uint64_t param) { const ValueType *type = (const ValueType *)(param); - CellsRef cells = getCellsRef(state.peek(0)); + TypedCells cells = getCellsRef(state.peek(0)); state.pop_push(state.stash.create<DenseTensorView>(*type, cells)); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp index c183e5c1db3..73cdfc12a38 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp @@ -21,68 +21,57 @@ calcCellsSize(const eval::ValueType &type) return cellsSize; } +template<typename T> void -checkCellsSize(const DenseTensor &arg) +checkCellsSize(const DenseTensor<T> &arg) { auto cellsSize = calcCellsSize(arg.fast_type()); - if (arg.cellsRef().size() != cellsSize) { + if (arg.cellsRef().size != cellsSize) { throw IllegalStateException(make_string("Wrong cell size, " "expected=%zu, " "actual=%zu", cellsSize, - arg.cellsRef().size())); + arg.cellsRef().size)); + } + if (arg.fast_type().cell_type() != arg.cellsRef().type) { + throw IllegalStateException(make_string("Wrong cell type, " + "expected=%u, " + "actual=%u", + (unsigned char)arg.fast_type().cell_type(), + (unsigned char)arg.cellsRef().type)); } } } -DenseTensor::DenseTensor() - : DenseTensorView(_type), - _type(eval::ValueType::double_type()), - _cells(1) -{ - initCellsRef(CellsRef(_cells)); -} - -DenseTensor::DenseTensor(const eval::ValueType &type_in, - const Cells &cells_in) - : DenseTensorView(_type), - _type(type_in), - _cells(cells_in) -{ - initCellsRef(CellsRef(_cells)); - checkCellsSize(*this); -} - - -DenseTensor::DenseTensor(const eval::ValueType &type_in, - Cells &&cells_in) - : DenseTensorView(_type), - _type(type_in), - _cells(std::move(cells_in)) -{ - initCellsRef(CellsRef(_cells)); - checkCellsSize(*this); -} - -DenseTensor::DenseTensor(eval::ValueType &&type_in, - Cells &&cells_in) +template <typename CT> +DenseTensor<CT>::DenseTensor(eval::ValueType type_in, + std::vector<CT> &&cells_in) : DenseTensorView(_type), _type(std::move(type_in)), _cells(std::move(cells_in)) { - initCellsRef(CellsRef(_cells)); + initCellsRef(TypedCells(_cells)); checkCellsSize(*this); } -DenseTensor::~DenseTensor() = default; +template <typename CT> +DenseTensor<CT>::~DenseTensor() = default; +template <typename CT> +template <typename RCT> bool -DenseTensor::operator==(const DenseTensor &rhs) const +DenseTensor<CT>::operator==(const DenseTensor<RCT> &rhs) const { - return (_type == rhs._type) && - (_cells == rhs._cells); + if (_type != rhs._type) return false; + if (_cells.size != rhs._cells.size) return false; + for (size_t i = 0; i < _cells.size; i++) { + if (_cells[i] != rhs._cells[i]) return false; + } + return true; } -} +template class DenseTensor<float>; +template class DenseTensor<double>; +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor.h b/eval/src/vespa/eval/tensor/dense/dense_tensor.h index 3795831c914..d0246fef635 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor.h @@ -10,20 +10,20 @@ namespace vespalib::tensor { * A dense tensor where all dimensions are indexed. * Tensor cells are stored in an underlying array according to the order of the dimensions. */ +template <typename CT> class DenseTensor : public DenseTensorView { public: - DenseTensor(); + DenseTensor() = delete; ~DenseTensor() override; - DenseTensor(const eval::ValueType &type_in, const Cells &cells_in); - DenseTensor(const eval::ValueType &type_in, Cells &&cells_in); - DenseTensor(eval::ValueType &&type_in, Cells &&cells_in); - bool operator==(const DenseTensor &rhs) const; + DenseTensor(eval::ValueType type_in, std::vector<CT> &&cells_in); + + // for unit tests + template <typename RCT> + bool operator==(const DenseTensor<RCT> &rhs) const; private: eval::ValueType _type; - Cells _cells; - + std::vector<CT> _cells; }; } - diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.cpp index 09bc546c982..c1c24d28b7f 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.cpp @@ -25,7 +25,7 @@ DenseTensorAddressMapper::mapLabelToNumber(stringref label) } uint32_t -DenseTensorAddressMapper::mapAddressToIndex(const TensorAddress &address, const eval::ValueType type) +DenseTensorAddressMapper::mapAddressToIndex(const TensorAddress &address, const eval::ValueType &type) { uint32_t idx = 0; TensorAddressElementIterator<TensorAddress> addressIterator(address); diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.h index cf20fc6ad3c..7cd776a260c 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.h @@ -21,7 +21,7 @@ public: static constexpr uint32_t BAD_LABEL = std::numeric_limits<uint32_t>::max(); static constexpr uint32_t BAD_ADDRESS = std::numeric_limits<uint32_t>::max(); static uint32_t mapLabelToNumber(stringref label); - static uint32_t mapAddressToIndex(const TensorAddress &address, const eval::ValueType type); + static uint32_t mapAddressToIndex(const TensorAddress &address, const eval::ValueType &type); }; } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp deleted file mode 100644 index e71840f392c..00000000000 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "dense_tensor_apply.h" -#include "dense_dimension_combiner.h" -#include "direct_dense_tensor_builder.h" - -namespace vespalib::tensor::dense { - -template <typename Function> -std::unique_ptr<Tensor> -apply(DenseDimensionCombiner & combiner, DirectDenseTensorBuilder & builder, - const DenseTensorView::CellsRef & lhsCells, - const DenseTensorView::CellsRef & rhsCells, Function &&func) __attribute__((noinline)); - -template <typename Function> -std::unique_ptr<Tensor> -apply(DenseDimensionCombiner & combiner, DirectDenseTensorBuilder & builder, - const DenseTensorView::CellsRef & lhsCells, - const DenseTensorView::CellsRef & rhsCells, Function &&func) -{ - for (combiner.leftReset(); combiner.leftInRange(); combiner.stepLeft()) { - for (combiner.rightReset(); combiner.rightInRange(); combiner.stepRight()) { - for (combiner.commonReset(); combiner.commonInRange(); combiner.stepCommon()) { - size_t outIdx = combiner.outputIdx(); - size_t l = combiner.leftIdx(); - size_t r = combiner.rightIdx(); - builder.insertCell(outIdx, func(lhsCells[l], rhsCells[r])); - } - } - } - return builder.build(); -} - -template <typename Function> -std::unique_ptr<Tensor> -apply(const DenseTensorView &lhs, const Tensor &rhs, Function &&func) -{ - const DenseTensorView *view = dynamic_cast<const DenseTensorView *>(&rhs); - if (view) { - DenseDimensionCombiner combiner(lhs.fast_type(), view->fast_type()); - DirectDenseTensorBuilder builder(combiner.result_type); - return apply(combiner, builder, lhs.cellsRef(), view->cellsRef(), std::move(func)); - } - return Tensor::UP(); -} - -} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp index 15c09db44b3..55d0e29bc35 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp @@ -4,7 +4,7 @@ namespace vespalib::tensor { -DenseTensorCellsIterator::DenseTensorCellsIterator(const eval::ValueType &type_in, CellsRef cells) +DenseTensorCellsIterator::DenseTensorCellsIterator(const eval::ValueType &type_in, TypedCells cells) : _type(type_in), _cells(cells), _cellIdx(0), diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h index caf92d6c8c7..8d189027be2 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h @@ -4,6 +4,7 @@ #include <vespa/eval/eval/value_type.h> #include <vespa/vespalib/util/arrayref.h> +#include "typed_cells.h" namespace vespalib::tensor { @@ -16,14 +17,14 @@ public: using size_type = eval::ValueType::Dimension::size_type; using Address = std::vector<size_type>; private: - using CellsRef = vespalib::ConstArrayRef<double>; + const eval::ValueType &_type; - CellsRef _cells; + TypedCells _cells; size_t _cellIdx; const int32_t _lastDimension; Address _address; public: - DenseTensorCellsIterator(const eval::ValueType &type_in, CellsRef cells); + DenseTensorCellsIterator(const eval::ValueType &type_in, TypedCells cells); ~DenseTensorCellsIterator(); void next() { ++_cellIdx; @@ -37,8 +38,8 @@ public: } } } - bool valid() const { return _cellIdx < _cells.size(); } - double cell() const { return _cells[_cellIdx]; } + bool valid() const { return _cellIdx < _cells.size; } + double cell() const { return _cells.get(_cellIdx); } const Address &address() const { return _address; } const eval::ValueType &fast_type() const { return _type; } }; diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_modify.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_modify.cpp index 4e2940f2516..4777abcbdef 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_modify.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_modify.cpp @@ -6,28 +6,37 @@ namespace vespalib::tensor { -DenseTensorModify::DenseTensorModify(join_fun_t op, const eval::ValueType &type, Cells cells) +template <class CT> +DenseTensorModify<CT>::DenseTensorModify(join_fun_t op, const eval::ValueType &type, std::vector<CT> &&cells) : _op(op), _type(type), _cells(std::move(cells)) { + assert(vespalib::eval::check_cell_type<CT>(type.cell_type())); } - -DenseTensorModify::~DenseTensorModify() = default; +template <class CT> +DenseTensorModify<CT>::~DenseTensorModify() = default; + +template <class CT> void -DenseTensorModify::visit(const TensorAddress &address, double value) +DenseTensorModify<CT>::visit(const TensorAddress &address, double value) { uint32_t idx = DenseTensorAddressMapper::mapAddressToIndex(address, _type); if (idx != DenseTensorAddressMapper::BAD_ADDRESS) { - _cells[idx] = _op(_cells[idx], value); + double nv = _op(_cells[idx], value); + _cells[idx] = (CT) nv; } } +template <class CT> std::unique_ptr<Tensor> -DenseTensorModify::build() +DenseTensorModify<CT>::build() { - return std::make_unique<DenseTensor>(std::move(_type), std::move(_cells)); + return std::make_unique<DenseTensor<CT>>(_type, std::move(_cells)); } -} +template class DenseTensorModify<float>; +template class DenseTensorModify<double>; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_modify.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_modify.h index 848e6e559c2..3d975c8595e 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_modify.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_modify.h @@ -12,16 +12,16 @@ namespace vespalib::tensor { * For all cells visited, a join function is applied to determine * the new cell value. */ +template <class CT> class DenseTensorModify : public TensorVisitor { using join_fun_t = Tensor::join_fun_t; - using Cells = DenseTensorView::Cells; join_fun_t _op; - eval::ValueType _type; - Cells _cells; + const eval::ValueType &_type; + std::vector<CT> _cells; public: - DenseTensorModify(join_fun_t op, const eval::ValueType &type, Cells cells); + DenseTensorModify(join_fun_t op, const eval::ValueType &type, std::vector<CT> &&cells); ~DenseTensorModify(); void visit(const TensorAddress &address, double value) override; std::unique_ptr<Tensor> build(); diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.cpp index 41643d8c266..252be199208 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.cpp @@ -4,10 +4,8 @@ namespace vespalib::tensor::dense { -namespace { - size_t -calcCellsSize(const eval::ValueType &type) +DimensionReducer::calcCellsSize(const eval::ValueType &type) { size_t cellsSize = 1; for (const auto &dim : type.dimensions()) { @@ -16,11 +14,9 @@ calcCellsSize(const eval::ValueType &type) return cellsSize; } -} DimensionReducer::DimensionReducer(const eval::ValueType &oldType, const string &dimensionToRemove) : _type(oldType.reduce({ dimensionToRemove })), - _cellsResult(calcCellsSize(_type)), _innerDimSize(1), _sumDimSize(1), _outerDimSize(1) diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp index 98db89dd2a7..5673a35c7e6 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp @@ -6,29 +6,27 @@ namespace vespalib::tensor::dense { -using Cells = DenseTensorView::Cells; -using CellsRef = DenseTensorView::CellsRef; - class DimensionReducer { private: eval::ValueType _type; - Cells _cellsResult; size_t _innerDimSize; size_t _sumDimSize; size_t _outerDimSize; + static size_t calcCellsSize(const eval::ValueType &type); void setup(const eval::ValueType &oldType, const vespalib::string &dimensionToRemove); - public: DimensionReducer(const eval::ValueType &oldType, const string &dimensionToRemove); ~DimensionReducer(); - template <typename Function> + template <typename T, typename Function> std::unique_ptr<DenseTensorView> - reduceCells(CellsRef cellsIn, Function &&func) { + reduceCells(ConstArrayRef<T> cellsIn, Function &&func) { + size_t resultSize = calcCellsSize(_type); + std::vector<T> cellsOut(resultSize); auto itr_in = cellsIn.cbegin(); - auto itr_out = _cellsResult.begin(); + auto itr_out = cellsOut.begin(); for (size_t outerDim = 0; outerDim < _outerDimSize; ++outerDim) { auto saved_itr = itr_out; for (size_t innerDim = 0; innerDim < _innerDimSize; ++innerDim) { @@ -45,20 +43,47 @@ public: } } } - assert(itr_out == _cellsResult.end()); + assert(itr_out == cellsOut.end()); assert(itr_in == cellsIn.cend()); - return std::make_unique<DenseTensor>(std::move(_type), std::move(_cellsResult)); + return std::make_unique<DenseTensor<T>>(std::move(_type), std::move(cellsOut)); } }; namespace { +struct CallReduceCells { + template <typename CT, typename Function> + static std::unique_ptr<DenseTensorView> + call(const ConstArrayRef<CT> &oldCells, DimensionReducer &reducer, Function &&func) { + return reducer.reduceCells(oldCells, func); + } + + template <typename CT, typename Function> + static double + call(const ConstArrayRef<CT> &oldCells, Function &&func) { + assert(oldCells.size() > 0); + double result = oldCells[0]; + for (size_t i = 1; i < oldCells.size(); ++i) { + result = func(result, oldCells[i]); + } + return result; + } +}; + template <typename Function> std::unique_ptr<DenseTensorView> reduce(const DenseTensorView &tensor, const vespalib::string &dimensionToRemove, Function &&func) { DimensionReducer reducer(tensor.fast_type(), dimensionToRemove); - return reducer.reduceCells(tensor.cellsRef(), func); + TypedCells oldCells = tensor.cellsRef(); + return dispatch_1<CallReduceCells>(oldCells, reducer, func); +} + +template <typename Function> +double +reduce_all_dimensions(TypedCells oldCells, Function &&func) +{ + return dispatch_1<CallReduceCells>(oldCells, func); } } @@ -67,18 +92,21 @@ template <typename Function> std::unique_ptr<Tensor> reduce(const DenseTensorView &tensor, const std::vector<vespalib::string> &dimensions, Function &&func) { - if (dimensions.size() == 1) { - return reduce(tensor, dimensions[0], func); - } else if (dimensions.size() > 0) { - std::unique_ptr<DenseTensorView> result = reduce(tensor, dimensions[0], func); - for (size_t i = 1; i < dimensions.size(); ++i) { - std::unique_ptr<DenseTensorView> tmpResult = reduce(*result, dimensions[i], func); - result = std::move(tmpResult); - } - return result; - } else { - return std::unique_ptr<Tensor>(); + if ((dimensions.size() == 0) || + (dimensions.size() == tensor.fast_type().dimensions().size())) + { + eval::ValueType newType = tensor.fast_type().reduce(dimensions); + assert(newType.is_double()); + double result = reduce_all_dimensions(tensor.cellsRef(), func); + std::vector<double> newCells({result}); + return std::make_unique<DenseTensor<double>>(std::move(newType), std::move(newCells)); + } + std::unique_ptr<DenseTensorView> result = reduce(tensor, dimensions[0], func); + for (size_t i = 1; i < dimensions.size(); ++i) { + std::unique_ptr<DenseTensorView> tmpResult = reduce(*result, dimensions[i], func); + result = std::move(tmpResult); } + return result; } } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp index 73b2e7b3ffb..d98cf52d279 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dense_tensor_view.h" -#include "dense_tensor_apply.hpp" +#include "dense_generic_join.hpp" #include "dense_tensor_reduce.hpp" #include "dense_tensor_modify.h" #include <vespa/vespalib/util/stringfmt.h> @@ -54,12 +54,12 @@ void checkCellsSize(const DenseTensorView &arg) { auto cellsSize = calcCellsSize(arg.fast_type()); - if (arg.cellsRef().size() != cellsSize) { + if (arg.cellsRef().size != cellsSize) { throw IllegalStateException(make_string("wrong cell size, " "expected=%zu, " "actual=%zu", cellsSize, - arg.cellsRef().size())); + arg.cellsRef().size)); } } @@ -67,7 +67,7 @@ void checkDimensions(const DenseTensorView &lhs, const DenseTensorView &rhs, vespalib::stringref operation) { - if (lhs.fast_type() != rhs.fast_type()) { + if (lhs.fast_type().dimensions() != rhs.fast_type().dimensions()) { throw IllegalStateException(make_string("mismatching dimensions for " "dense tensor %s, " "lhs dimensions = '%s', " @@ -87,24 +87,52 @@ checkDimensions(const DenseTensorView &lhs, const DenseTensorView &rhs, * The given function is used to calculate the resulting cell value * for overlapping cells. */ +template <typename LCT, typename RCT, typename Function> +static Tensor::UP +sameShapeJoin(const ConstArrayRef<LCT> &lhs, const ConstArrayRef<RCT> &rhs, + const eval::ValueType &lhs_type, + Function &&func) +{ + size_t sz = lhs.size(); + assert(sz == rhs.size()); + using OutputSelector = OutputCellType<LCT, RCT>; + using OCT = typename OutputSelector::output_type; + std::vector<OCT> newCells; + newCells.reserve(sz); + auto rhsCellItr = rhs.cbegin(); + for (const auto &lhsCell : lhs) { + OCT v = func(lhsCell, *rhsCellItr); + newCells.push_back(v); + ++rhsCellItr; + } + assert(rhsCellItr == rhs.cend()); + assert(newCells.size() == sz); + auto newType = eval::ValueType::tensor_type(lhs_type.dimensions(), OutputSelector::output_cell_type()); + return std::make_unique<DenseTensor<OCT>>(std::move(newType), std::move(newCells)); +} + +struct CallJoin +{ + template <typename LCT, typename RCT, typename Function> + static Tensor::UP + call(const ConstArrayRef<LCT> &lhs, const ConstArrayRef<RCT> &rhs, + const eval::ValueType &lhs_type, + Function &&func) + { + return sameShapeJoin(lhs, rhs, lhs_type, std::move(func)); + } +}; + template <typename Function> Tensor::UP joinDenseTensors(const DenseTensorView &lhs, const DenseTensorView &rhs, Function &&func) { - DenseTensor::Cells cells; - cells.reserve(lhs.cellsRef().size()); - auto rhsCellItr = rhs.cellsRef().cbegin(); - for (const auto &lhsCell : lhs.cellsRef()) { - cells.push_back(func(lhsCell, *rhsCellItr)); - ++rhsCellItr; - } - assert(rhsCellItr == rhs.cellsRef().cend()); - return std::make_unique<DenseTensor>(lhs.fast_type(), - std::move(cells)); + TypedCells lhsCells = lhs.cellsRef(); + TypedCells rhsCells = rhs.cellsRef(); + return dispatch_2<CallJoin>(lhsCells, rhsCells, lhs.fast_type(), std::move(func)); } - template <typename Function> Tensor::UP joinDenseTensors(const DenseTensorView &lhs, const Tensor &rhs, @@ -119,13 +147,13 @@ joinDenseTensors(const DenseTensorView &lhs, const Tensor &rhs, return Tensor::UP(); } -bool sameCells(DenseTensorView::CellsRef lhs, DenseTensorView::CellsRef rhs) +bool sameCells(TypedCells lhs, TypedCells rhs) { - if (lhs.size() != rhs.size()) { + if (lhs.size != rhs.size) { return false; } - for (size_t i = 0; i < lhs.size(); ++i) { - if (lhs[i] != rhs[i]) { + for (size_t i = 0; i < lhs.size; ++i) { + if (lhs.get(i) != rhs.get(i)) { return false; } } @@ -146,27 +174,43 @@ DenseTensorView::type() const return _typeRef; } +struct CallSum { + template <typename CT> + static double + call(const ConstArrayRef<CT> &arr) { + double res = 0.0; + for (CT val : arr) { + res += val; + } + return res; + } +}; + double DenseTensorView::as_double() const { - double result = 0.0; - for (const auto &cell : _cellsRef) { - result += cell; - } - return result; + return dispatch_1<CallSum>(_cellsRef); } +struct CallApply { + template <typename CT> + static Tensor::UP + call(const ConstArrayRef<CT> &oldCells, const eval::ValueType &newType, const CellFunction &func) + { + std::vector<CT> newCells; + newCells.reserve(oldCells.size()); + for (const auto &cell : oldCells) { + CT nv = func.apply(cell); + newCells.push_back(nv); + } + return std::make_unique<DenseTensor<CT>>(newType, std::move(newCells)); + } +}; + Tensor::UP DenseTensorView::apply(const CellFunction &func) const { - Cells newCells(_cellsRef.size()); - auto itr = newCells.begin(); - for (const auto &cell : _cellsRef) { - *itr = func.apply(cell); - ++itr; - } - assert(itr == newCells.end()); - return std::make_unique<DenseTensor>(_typeRef, std::move(newCells)); + return dispatch_1<CallApply>(_cellsRef, _typeRef, func); } bool @@ -179,11 +223,20 @@ DenseTensorView::equals(const Tensor &arg) const return false; } +struct CallClone { + template<class CT> + static Tensor::UP + call(const ConstArrayRef<CT> &cells, eval::ValueType newType) + { + std::vector<CT> newCells(cells.begin(), cells.end()); + return std::make_unique<DenseTensor<CT>>(std::move(newType), std::move(newCells)); + } +}; + Tensor::UP DenseTensorView::clone() const { - return std::make_unique<DenseTensor>(_typeRef, - Cells(_cellsRef.cbegin(), _cellsRef.cend())); + return dispatch_1<CallClone>(_cellsRef, _typeRef); } namespace { @@ -249,12 +302,12 @@ DenseTensorView::join(join_fun_t function, const Tensor &arg) const return joinDenseTensors(*this, arg, "join", function); } if (function == eval::operation::Mul::f) { - return dense::apply(*this, arg, [](double a, double b) { return (a * b); }); + return dense::generic_join(*this, arg, [](double a, double b) { return (a * b); }); } if (function == eval::operation::Add::f) { - return dense::apply(*this, arg, [](double a, double b) { return (a + b); }); + return dense::generic_join(*this, arg, [](double a, double b) { return (a + b); }); } - return dense::apply(*this, arg, function); + return dense::generic_join(*this, arg, function); } Tensor::UP @@ -277,12 +330,25 @@ DenseTensorView::reduce(join_fun_t op, const std::vector<vespalib::string> &dime : reduce_all(op, dimensions); } +struct CallModify +{ + using join_fun_t = DenseTensorView::join_fun_t; + + template <typename CT> + static std::unique_ptr<Tensor> + call(const ConstArrayRef<CT> &arr, join_fun_t op, const eval::ValueType &typeRef, const CellValues &cellValues) + { + std::vector newCells(arr.begin(), arr.end()); + DenseTensorModify<CT> modifier(op, typeRef, std::move(newCells)); + cellValues.accept(modifier); + return modifier.build(); + } +}; + std::unique_ptr<Tensor> DenseTensorView::modify(join_fun_t op, const CellValues &cellValues) const { - DenseTensorModify modifier(op, _typeRef, Cells(_cellsRef.cbegin(), _cellsRef.cend())); - cellValues.accept(modifier); - return modifier.build(); + return dispatch_1<CallModify>(_cellsRef, op, _typeRef, cellValues); } std::unique_ptr<Tensor> diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h index 09b6b72375e..60f85c38659 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h @@ -2,6 +2,7 @@ #pragma once +#include "typed_cells.h" #include "dense_tensor_cells_iterator.h" #include <vespa/eval/tensor/tensor.h> @@ -14,12 +15,10 @@ namespace vespalib::tensor { class DenseTensorView : public Tensor { public: - using Cells = std::vector<double>; - using CellsRef = ConstArrayRef<double>; using CellsIterator = DenseTensorCellsIterator; using Address = std::vector<eval::ValueType::Dimension::size_type>; - DenseTensorView(const eval::ValueType &type_in, CellsRef cells_in) + DenseTensorView(const eval::ValueType &type_in, TypedCells cells_in) : _typeRef(type_in), _cellsRef(cells_in) {} @@ -29,7 +28,7 @@ public: {} const eval::ValueType &fast_type() const { return _typeRef; } - const CellsRef &cellsRef() const { return _cellsRef; } + const TypedCells &cellsRef() const { return _cellsRef; } bool operator==(const DenseTensorView &rhs) const; CellsIterator cellsIterator() const { return CellsIterator(_typeRef, _cellsRef); } @@ -46,14 +45,14 @@ public: eval::TensorSpec toSpec() const override; void accept(TensorVisitor &visitor) const override; protected: - void initCellsRef(CellsRef cells_in) { + void initCellsRef(TypedCells cells_in) { _cellsRef = cells_in; } private: Tensor::UP reduce_all(join_fun_t op, const std::vector<vespalib::string> &dimensions) const; const eval::ValueType &_typeRef; - CellsRef _cellsRef; + TypedCells _cellsRef; }; } diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp index a3056311fab..b6ac87ce012 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp @@ -12,7 +12,6 @@ namespace vespalib::tensor { -using CellsRef = DenseTensorView::CellsRef; using eval::ValueType; using eval::TensorFunction; using eval::as; @@ -22,9 +21,11 @@ using namespace eval::operation; namespace { -CellsRef getCellsRef(const eval::Value &value) { +XWInput getCellsRef(const eval::Value &value) { const DenseTensorView &denseTensor = static_cast<const DenseTensorView &>(value); - return denseTensor.cellsRef(); + TypedCells ref = denseTensor.cellsRef(); + assert(ref.type == CellType::DOUBLE); + return ref.typify<double>(); } void multiDotProduct(const DenseXWProductFunction::Self &self, @@ -62,8 +63,8 @@ template <bool commonDimensionInnermost> void my_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { DenseXWProductFunction::Self *self = (DenseXWProductFunction::Self *)(param); - CellsRef vectorCells = getCellsRef(state.peek(1)); - CellsRef matrixCells = getCellsRef(state.peek(0)); + XWInput vectorCells = getCellsRef(state.peek(1)); + XWInput matrixCells = getCellsRef(state.peek(0)); ArrayRef<double> outputCells = state.stash.create_array<double>(self->_resultSize); @@ -72,7 +73,7 @@ void my_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { } else { transposedProduct(*self, vectorCells, matrixCells, outputCells); } - state.pop_pop_push(state.stash.create<DenseTensorView>(self->_resultType, outputCells)); + state.pop_pop_push(state.stash.create<DenseTensorView>(self->_resultType, TypedCells(outputCells))); } bool isConcreteDenseTensor(const ValueType &type, size_t d) { diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h index a8ccdd331cf..9f1bc12b110 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h @@ -8,7 +8,7 @@ namespace vespalib::tensor { -using XWInput = DenseTensorView::CellsRef; +using XWInput = ConstArrayRef<double>; using XWOutput = ArrayRef<double>; /** diff --git a/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.cpp b/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.cpp deleted file mode 100644 index f800750bf8f..00000000000 --- a/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "direct_dense_tensor_builder.h" - -namespace vespalib::tensor { - -using Address = DirectDenseTensorBuilder::Address; -using eval::ValueType; - -namespace { - -size_t -calculateCellsSize(const ValueType &type) -{ - size_t cellsSize = 1; - for (const auto &dim : type.dimensions()) { - cellsSize *= dim.size; - } - return cellsSize; -} - -} - -DirectDenseTensorBuilder::~DirectDenseTensorBuilder() = default; - -DirectDenseTensorBuilder::DirectDenseTensorBuilder(const ValueType &type_in) - : _type(type_in), - _cells(calculateCellsSize(_type)) -{ -} - -Tensor::UP -DirectDenseTensorBuilder::build() -{ - return std::make_unique<DenseTensor>(std::move(_type), std::move(_cells)); -} - -} - diff --git a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp index 79a8b994480..08abc391179 100644 --- a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp @@ -21,12 +21,12 @@ MutableDenseTensorView::MutableValueType::MutableValueType(ValueType type_in) MutableDenseTensorView::MutableValueType::~MutableValueType() = default; MutableDenseTensorView::MutableDenseTensorView(ValueType type_in) - : DenseTensorView(_concreteType._type, CellsRef()), + : DenseTensorView(_concreteType._type), _concreteType(type_in) { } -MutableDenseTensorView::MutableDenseTensorView(ValueType type_in, CellsRef cells_in) +MutableDenseTensorView::MutableDenseTensorView(ValueType type_in, TypedCells cells_in) : DenseTensorView(_concreteType._type, cells_in), _concreteType(type_in) { diff --git a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h index 260e71b6f76..d71903d6c47 100644 --- a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h +++ b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h @@ -42,8 +42,8 @@ private: public: MutableDenseTensorView(eval::ValueType type_in); - MutableDenseTensorView(eval::ValueType type_in, CellsRef cells_in); - void setCells(CellsRef cells_in) { + MutableDenseTensorView(eval::ValueType type_in, TypedCells cells_in); + void setCells(TypedCells cells_in) { initCellsRef(cells_in); } void setUnboundDimensions(const uint32_t *unboundDimSizeBegin, const uint32_t *unboundDimSizeEnd) { diff --git a/eval/src/vespa/eval/tensor/dense/typed_cells.cpp b/eval/src/vespa/eval/tensor/dense/typed_cells.cpp new file mode 100644 index 00000000000..e56325bffd2 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/typed_cells.cpp @@ -0,0 +1,7 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "typed_cells.h" + +namespace vespalib::tensor { + +} // namespace diff --git a/eval/src/vespa/eval/tensor/dense/typed_cells.h b/eval/src/vespa/eval/tensor/dense/typed_cells.h new file mode 100644 index 00000000000..d1b6058bfbe --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/typed_cells.h @@ -0,0 +1,96 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <assert.h> +#include <vespa/vespalib/util/arrayref.h> +#include <vespa/eval/eval/value_type.h> + +namespace vespalib::tensor { + +// Low-level typed cells reference + +using CellType = vespalib::eval::ValueType::CellType; + + +template<typename LCT, typename RCT> struct OutputCellType; +template<> struct OutputCellType<double, double> { + typedef double output_type; + static constexpr CellType output_cell_type() { return CellType::DOUBLE; }; +}; +template<> struct OutputCellType<float, double> { + typedef double output_type; + static constexpr CellType output_cell_type() { return CellType::DOUBLE; }; +}; +template<> struct OutputCellType<double, float> { + typedef double output_type; + static constexpr CellType output_cell_type() { return CellType::DOUBLE; }; +}; +template<> struct OutputCellType<float, float> { + typedef float output_type; + static constexpr CellType output_cell_type() { return CellType::FLOAT; }; +}; + +struct TypedCells { + const void *data; + CellType type; + size_t size:56; + + explicit TypedCells(ConstArrayRef<double> cells) : data(cells.begin()), type(CellType::DOUBLE), size(cells.size()) {} + explicit TypedCells(ConstArrayRef<float> cells) : data(cells.begin()), type(CellType::FLOAT), size(cells.size()) {} + + TypedCells() : data(nullptr), type(CellType::DOUBLE), size(0) {} + TypedCells(const void *dp, CellType ct, size_t sz) : data(dp), type(ct), size(sz) {} + + template <typename T> bool check_type() const { return vespalib::eval::check_cell_type<T>(type); } + template <typename T> ConstArrayRef<T> typify() const { + assert(check_type<T>()); + return ConstArrayRef<T>((const T *)data, size); + } + template <typename T> ConstArrayRef<T> unsafe_typify() const { + return ConstArrayRef<T>((const T *)data, size); + } + + double get(size_t idx) const { + if (type == CellType::DOUBLE) { + const double *p = (const double *)data; + return p[idx]; + } + if (type == CellType::FLOAT) { + const float *p = (const float *)data; + return p[idx]; + } + abort(); + } + + TypedCells & operator= (const TypedCells &other) = default; +}; + +template <typename TGT, typename... Args> +auto dispatch_0(CellType ct, Args &&...args) { + switch (ct) { + case CellType::DOUBLE: return TGT::template call<double>(std::forward<Args>(args)...); + case CellType::FLOAT: return TGT::template call<float>(std::forward<Args>(args)...); + } + abort(); +} + +template <typename TGT, typename... Args> +auto dispatch_1(const TypedCells &a, Args &&...args) { + switch (a.type) { + case CellType::DOUBLE: return TGT::call(a.unsafe_typify<double>(), std::forward<Args>(args)...); + case CellType::FLOAT: return TGT::call(a.unsafe_typify<float>(), std::forward<Args>(args)...); + } + abort(); +} + +template <typename TGT, typename A1, typename... Args> +auto dispatch_2(A1 &&a, const TypedCells &b, Args &&...args) { + switch (b.type) { + case CellType::DOUBLE: return dispatch_1<TGT>(std::forward<A1>(a), b.unsafe_typify<double>(), std::forward<Args>(args)...); + case CellType::FLOAT: return dispatch_1<TGT>(std::forward<A1>(a), b.unsafe_typify<float>(), std::forward<Args>(args)...); + } + abort(); +} + +} // namespace diff --git a/eval/src/vespa/eval/tensor/dense/typed_dense_tensor_builder.cpp b/eval/src/vespa/eval/tensor/dense/typed_dense_tensor_builder.cpp new file mode 100644 index 00000000000..3e7d234fd54 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/typed_dense_tensor_builder.cpp @@ -0,0 +1,45 @@ + + +#include "typed_dense_tensor_builder.h" + +namespace vespalib::tensor { + +using Address = DenseTensorView::Address; +using eval::ValueType; + +namespace { + +size_t +calculateCellsSize(const ValueType &type) +{ + size_t cellsSize = 1; + for (const auto &dim : type.dimensions()) { + cellsSize *= dim.size; + } + return cellsSize; +} + +} // namespace + +template <typename CT> +TypedDenseTensorBuilder<CT>::~TypedDenseTensorBuilder() = default; + +template <typename CT> +TypedDenseTensorBuilder<CT>::TypedDenseTensorBuilder(const ValueType &type_in) + : _type(type_in), + _cells(calculateCellsSize(_type)) +{ + assert(vespalib::eval::check_cell_type<CT>(_type.cell_type())); +} + +template <typename CT> +Tensor::UP +TypedDenseTensorBuilder<CT>::build() +{ + return std::make_unique<DenseTensor<CT>>(std::move(_type), std::move(_cells)); +} + +template class TypedDenseTensorBuilder<double>; +template class TypedDenseTensorBuilder<float>; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.h b/eval/src/vespa/eval/tensor/dense/typed_dense_tensor_builder.h index 935b5c20373..770ea4ae5ea 100644 --- a/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.h +++ b/eval/src/vespa/eval/tensor/dense/typed_dense_tensor_builder.h @@ -9,15 +9,14 @@ namespace vespalib::tensor { /** * Class for building a dense tensor by inserting cell values directly into underlying array of cells. */ -class DirectDenseTensorBuilder +template <typename CT> +class TypedDenseTensorBuilder { public: - using Cells = DenseTensor::Cells; - using Address = DenseTensor::Address; - + using Address = DenseTensorView::Address; private: eval::ValueType _type; - Cells _cells; + std::vector<CT> _cells; static size_t calculateCellAddress(const Address &address, const eval::ValueType &type) { size_t result = 0; @@ -28,16 +27,15 @@ private: return result; } public: - DirectDenseTensorBuilder(const eval::ValueType &type_in); - ~DirectDenseTensorBuilder(); - void insertCell(const Address &address, double cellValue) { + TypedDenseTensorBuilder(const eval::ValueType &type_in); + ~TypedDenseTensorBuilder(); + void insertCell(const Address &address, CT cellValue) { insertCell(calculateCellAddress(address, _type), cellValue); } - void insertCell(size_t index, double cellValue) { + void insertCell(size_t index, CT cellValue) { _cells[index] = cellValue; } Tensor::UP build(); }; } - diff --git a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp index 445b08ab114..4694aea717d 100644 --- a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp @@ -9,7 +9,6 @@ namespace vespalib::tensor { -using CellsRef = DenseTensorView::CellsRef; using eval::Value; using eval::ValueType; using eval::TensorFunction; @@ -20,14 +19,25 @@ using namespace eval::operation; namespace { +struct CallVectorFromDoubles { + template <typename CT> + static TypedCells + call(eval::InterpretedFunction::State &state, size_t numCells) { + ArrayRef<CT> outputCells = state.stash.create_array<CT>(numCells); + for (size_t i = numCells; i-- > 0; ) { + outputCells[i] = (CT) state.peek(0).as_double(); + state.stack.pop_back(); + } + return TypedCells(outputCells); + } +}; + void my_vector_from_doubles_op(eval::InterpretedFunction::State &state, uint64_t param) { const auto *self = (const VectorFromDoublesFunction::Self *)(param); - ArrayRef<double> outputCells = state.stash.create_array<double>(self->resultSize); - for (size_t i = self->resultSize; i-- > 0; ) { - outputCells[i] = state.peek(0).as_double(); - state.stack.pop_back(); - } - const Value &result = state.stash.create<DenseTensorView>(self->resultType, outputCells); + CellType ct = self->resultType.cell_type(); + size_t numCells = self->resultSize; + TypedCells cells = dispatch_0<CallVectorFromDoubles>(ct, state, numCells); + const Value &result = state.stash.create<DenseTensorView>(self->resultType, cells); state.stack.push_back(result); } diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp index 677fb40b0f4..493e4af3caf 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp @@ -29,9 +29,10 @@ size_t encodeDimensions(nbostream &stream, const eval::ValueType & type) { } template<typename T> -void encodeCells(nbostream &stream, DenseTensorView::CellsRef cells) { - for (const auto &value : cells) { - stream << static_cast<T>(value); +void encodeCells(nbostream &stream, TypedCells cells) { + auto arr = cells.typify<T>(); + for (const auto &value : arr) { + stream << value; } } @@ -76,8 +77,8 @@ void DenseBinaryFormat::serialize(nbostream &stream, const DenseTensorView &tensor) { size_t cellsSize = encodeDimensions(stream, tensor.fast_type()); - DenseTensorView::CellsRef cells = tensor.cellsRef(); - assert(cells.size() == cellsSize); + TypedCells cells = tensor.cellsRef(); + assert(cells.size == cellsSize); switch (tensor.fast_type().cell_type()) { case CellType::DOUBLE: encodeCells<double>(stream, cells); @@ -88,15 +89,24 @@ DenseBinaryFormat::serialize(nbostream &stream, const DenseTensorView &tensor) } } -std::unique_ptr<DenseTensor> +struct CallDecodeCells { + template <typename CT> + static std::unique_ptr<DenseTensorView> + call(nbostream &stream, size_t numCells, ValueType &&newType) { + std::vector<CT> newCells; + newCells.reserve(numCells); + decodeCells<CT>(stream, numCells, newCells); + return std::make_unique<DenseTensor<CT>>(std::move(newType), std::move(newCells)); + } +}; + +std::unique_ptr<DenseTensorView> DenseBinaryFormat::deserialize(nbostream &stream, CellType cell_type) { std::vector<Dimension> dimensions; - size_t cellsSize = decodeDimensions(stream,dimensions); - DenseTensor::Cells cells; - cells.reserve(cellsSize); - decodeCells(cell_type, stream, cellsSize, cells); - return std::make_unique<DenseTensor>(ValueType::tensor_type(std::move(dimensions), cell_type), std::move(cells)); + size_t numCells = decodeDimensions(stream, dimensions); + ValueType newType = ValueType::tensor_type(std::move(dimensions), cell_type); + return dispatch_0<CallDecodeCells>(cell_type, stream, numCells, std::move(newType)); } template <typename T> @@ -104,7 +114,7 @@ void DenseBinaryFormat::deserializeCellsOnly(nbostream &stream, std::vector<T> &cells, CellType cell_type) { std::vector<Dimension> dimensions; - size_t cellsSize = decodeDimensions(stream,dimensions); + size_t cellsSize = decodeDimensions(stream, dimensions); cells.clear(); cells.reserve(cellsSize); decodeCells(cell_type, stream, cellsSize, cells); diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h index 9e860b3c1e4..21618dcb6ce 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h @@ -10,7 +10,6 @@ namespace vespalib { class nbostream; } namespace vespalib::tensor { -class DenseTensor; class DenseTensorView; /** @@ -22,7 +21,7 @@ public: using CellType = eval::ValueType::CellType; static void serialize(nbostream &stream, const DenseTensorView &tensor); - static std::unique_ptr<DenseTensor> deserialize(nbostream &stream, CellType cell_type); + static std::unique_ptr<DenseTensorView> deserialize(nbostream &stream, CellType cell_type); // This is a temporary method untill we get full support for typed tensors template <typename T> diff --git a/eval/src/vespa/eval/tensor/tensor_mapper.cpp b/eval/src/vespa/eval/tensor/tensor_mapper.cpp deleted file mode 100644 index 6f2b094af9e..00000000000 --- a/eval/src/vespa/eval/tensor/tensor_mapper.cpp +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tensor_mapper.h" -#include "tensor.h" -#include "tensor_visitor.h" -#include "tensor_address_element_iterator.h" -#include "wrapped_simple_tensor.h" -#include <vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h> -#include <vespa/eval/tensor/dense/dense_tensor.h> -#include <vespa/eval/tensor/dense/dense_tensor_address_mapper.h> -#include <vespa/vespalib/stllike/hash_map.hpp> -#include <limits> - -using vespalib::eval::ValueType; -using vespalib::eval::TensorSpec; - -namespace vespalib::tensor { - -namespace { - -//----------------------------------------------------------------------------- - -template <class TensorT> -class SparseTensorMapper : public TensorVisitor -{ - using Builder = DirectSparseTensorBuilder; - using AddressBuilderType = typename Builder::AddressBuilderType; - - Builder _builder; - AddressBuilderType _addressBuilder; - - void mapAddress(const TensorAddress &address); - virtual void visit(const TensorAddress &address, double value) override; - - SparseTensorMapper(const ValueType &type); - - ~SparseTensorMapper(); - - std::unique_ptr<Tensor> build(); -public: - static std::unique_ptr<Tensor> - map(const Tensor &tensor, const ValueType &type); -}; - -template <class TensorT> -SparseTensorMapper<TensorT>:: -SparseTensorMapper(const ValueType &type) - : TensorVisitor(), - _builder(type), - _addressBuilder() -{ -} - -template <class TensorT> -SparseTensorMapper<TensorT>::~SparseTensorMapper() = default; - -template <class TensorT> -std::unique_ptr<Tensor> -SparseTensorMapper<TensorT>::build() -{ - return _builder.build(); -} - -template <> -void -SparseTensorMapper<SparseTensor>:: -mapAddress(const TensorAddress &address) -{ - _addressBuilder.clear(); - TensorAddressElementIterator<TensorAddress> addressIterator(address); - for (const auto &dimension : _builder.fast_type().dimensions()) { - if (addressIterator.skipToDimension(dimension.name)) { - _addressBuilder.add(addressIterator.label()); - addressIterator.next(); - } else { - // output dimension not in input - _addressBuilder.addUndefined(); - } - } -} - -template <class TensorT> -void -SparseTensorMapper<TensorT>::visit(const TensorAddress &address, double value) -{ - mapAddress(address); - _builder.insertCell(_addressBuilder, value, - [](double oldValue, double newValue) - { return oldValue + newValue; }); -} - -template <class TensorT> -std::unique_ptr<Tensor> -SparseTensorMapper<TensorT>::map(const Tensor &tensor, - const ValueType &type) -{ - SparseTensorMapper<TensorT> mapper(type); - tensor.accept(mapper); - return mapper.build(); -} - -//----------------------------------------------------------------------------- - -class DenseTensorMapper : public TensorVisitor -{ - ValueType _type; - DenseTensor::Cells _cells; - - uint32_t mapAddressToIndex(const TensorAddress &address); - virtual void visit(const TensorAddress &address, double value) override; - - DenseTensorMapper(const ValueType &type); - ~DenseTensorMapper(); - - std::unique_ptr<Tensor> build(); -public: - static std::unique_ptr<Tensor> - map(const Tensor &tensor, const ValueType &type); -}; - -DenseTensorMapper::DenseTensorMapper(const ValueType &type) - : _type(type), - _cells() -{ - size_t size = 1; - for (const auto &dimension : type.dimensions()) { - size *= dimension.size; - } - _cells.resize(size); -} - -DenseTensorMapper::~DenseTensorMapper() -{ -} - -std::unique_ptr<Tensor> -DenseTensorMapper::build() -{ - return std::make_unique<DenseTensor>(std::move(_type), - std::move(_cells)); -} - -void -DenseTensorMapper::visit(const TensorAddress &address, double value) -{ - uint32_t idx = DenseTensorAddressMapper::mapAddressToIndex(address, _type); - if (idx != DenseTensorAddressMapper::BAD_ADDRESS) { - assert(idx < _cells.size()); - _cells[idx] += value; - } -} - -std::unique_ptr<Tensor> -DenseTensorMapper::map(const Tensor &tensor, const ValueType &type) -{ - DenseTensorMapper mapper(type); - tensor.accept(mapper); - return mapper.build(); -} - -//----------------------------------------------------------------------------- - -class WrappedTensorMapper : public TensorVisitor -{ - using Label = TensorSpec::Label; - - ValueType _type; - TensorSpec _spec; - - WrappedTensorMapper(const ValueType &type) - : _type(type), _spec(type.to_spec()) {} - ~WrappedTensorMapper() {} - - void visit(const TensorAddress &address, double value) override; - - std::unique_ptr<Tensor> build() { - auto tensor = eval::SimpleTensor::create(_spec); - return std::make_unique<WrappedSimpleTensor>(std::move(tensor)); - } - -public: - static std::unique_ptr<Tensor> - map(const Tensor &tensor, const ValueType &type); -}; - -void -WrappedTensorMapper::visit(const TensorAddress &address, double value) -{ - TensorSpec::Address addr; - TensorAddressElementIterator<TensorAddress> addressIterator(address); - for (const auto &dimension: _type.dimensions()) { - if (addressIterator.skipToDimension(dimension.name)) { - if (dimension.is_indexed()) { - uint32_t label = DenseTensorAddressMapper::mapLabelToNumber(addressIterator.label()); - if ((label == DenseTensorAddressMapper::BAD_LABEL) || (label >= dimension.size)) { - return; // bad address; ignore cell - } - addr.emplace(dimension.name, label); - } else { - addr.emplace(dimension.name, addressIterator.label()); - } - addressIterator.next(); - } else { - if (dimension.is_indexed()) { - addr.emplace(dimension.name, size_t(0)); - } else { - addr.emplace(dimension.name, vespalib::string()); - } - } - } - _spec.add(addr, value); -} - -std::unique_ptr<Tensor> -WrappedTensorMapper::map(const Tensor &tensor, const ValueType &type) -{ - WrappedTensorMapper mapper(type); - tensor.accept(mapper); - return mapper.build(); -} - -//----------------------------------------------------------------------------- - -} // namespace vespalib::tensor::<anonymous> - -TensorMapper::TensorMapper(const ValueType &type) - : _type(type) -{ -} - -TensorMapper::~TensorMapper() -{ -} - -template <typename TensorT> -std::unique_ptr<Tensor> -TensorMapper::mapToSparse(const Tensor &tensor, const ValueType &type) -{ - assert(type.is_sparse()); - return SparseTensorMapper<TensorT>::map(tensor, type); -} - -std::unique_ptr<Tensor> -TensorMapper::mapToDense(const Tensor &tensor, const ValueType &type) -{ - assert(type.is_dense()); - return DenseTensorMapper::map(tensor, type); -} - -std::unique_ptr<Tensor> -TensorMapper::mapToWrapped(const Tensor &tensor, const ValueType &type) -{ - assert(!type.dimensions().empty()); - return WrappedTensorMapper::map(tensor, type); -} - -std::unique_ptr<Tensor> -TensorMapper::map(const Tensor &tensor) const -{ - if (_type.is_sparse()) { - return mapToSparse<SparseTensor>(tensor, _type); - } else if (_type.is_dense()) { - return mapToDense(tensor, _type); - } else { - return mapToWrapped(tensor, _type); - } -} - -template -std::unique_ptr<Tensor> -TensorMapper::mapToSparse<SparseTensor>(const Tensor &tensor, - const ValueType &type); - -} diff --git a/eval/src/vespa/eval/tensor/tensor_mapper.h b/eval/src/vespa/eval/tensor/tensor_mapper.h deleted file mode 100644 index 95c6cce8fc6..00000000000 --- a/eval/src/vespa/eval/tensor/tensor_mapper.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/eval/eval/value_type.h> - -namespace vespalib::tensor { - -class Tensor; - -/** - * Class to map a tensor to a given tensor type. Dimensions in input - * tensor not present in tensor type are ignored. Dimensions in tensor - * type not present in input tensor gets default label (undefined - * (empty string) for sparse tensors, 0 for dense tensors). Values are - * accumulated for identical mapped addresses. - * - * Dense tensor type has further restrictions: label must contain only - * numerical digits (0-9). Empty string equals 0. If the label is - * parsed to a value outside the dimension range or the parsing fails, - * then the cell ((address, value) pair) is ignored. - */ -class TensorMapper -{ - eval::ValueType _type; -public: - TensorMapper(const eval::ValueType &type); - ~TensorMapper(); - - template <typename TensorT> - static std::unique_ptr<Tensor> - mapToSparse(const Tensor &tensor, const eval::ValueType &type); - - static std::unique_ptr<Tensor> - mapToDense(const Tensor &tensor, const eval::ValueType &type); - - static std::unique_ptr<Tensor> - mapToWrapped(const Tensor &tensor, const eval::ValueType &type); - - std::unique_ptr<Tensor> map(const Tensor &tensor) const; -}; - - -} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index b969e328419..5e9c0e2543c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -151,6 +151,12 @@ public class Flags { "Takes effect on deployment through controller", APPLICATION_ID); + public static final UnboundBooleanFlag MULTIPLE_GLOBAL_ENDPOINTS = defineFeatureFlag( + "multiple-global-endpoints", false, + "Allow applications to use new endpoints syntax in deployment.xml", + "Takes effect on deployment through controller", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java index 764029f18e0..0b392ebfa03 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java @@ -13,6 +13,7 @@ public final class Base64DecodeExpression extends Expression { public Base64DecodeExpression() { super(DataType.STRING); } + @Override protected void doExecute(ExecutionContext ctx) { String input = String.valueOf(ctx.getValue()); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java index 816d6a62fd0..d91338e3d3f 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java @@ -48,6 +48,10 @@ public final class NGramExpression extends Expression { @Override protected void doExecute(ExecutionContext ctx) { StringFieldValue input = (StringFieldValue)ctx.getValue(); + if (input.getSpanTree(SpanTrees.LINGUISTICS) != null) { + // This expression is already executed for this input instance + return; + } SpanList spanList = input.setSpanTree(new SpanTree(SpanTrees.LINGUISTICS)).spanList(); int lastPosition = 0; for (Iterator<GramSplitter.Gram> it = linguistics.getGramSplitter().split(input.getString(), gramSize); it.hasNext();) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java index bad1407c7c1..0b217d5ba9a 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java @@ -77,6 +77,22 @@ public class NGramTestCase { assertFalse(i.hasNext()); } + @Test + public void requireThatExecuteCanBeCalledMultipleTimes() { + ExecutionContext context = new ExecutionContext(new SimpleTestAdapter()); + context.setValue(new StringFieldValue("some random text string")); + NGramExpression expression = new NGramExpression(new SimpleLinguistics(), 3); + + expression.execute(context); + SpanTree firstTree = ((StringFieldValue)context.getValue()).getSpanTree(SpanTrees.LINGUISTICS); + assertNotNull(firstTree); + + expression.execute(context); + SpanTree secondTree = ((StringFieldValue)context.getValue()).getSpanTree(SpanTrees.LINGUISTICS); + // The span tree instance should be the same. + assertEquals(firstTree, secondTree); + } + private void assertSpan(int from, int length, boolean gram, Iterator<SpanNode> i, SpanTree tree) { assertSpan(from, length, gram, i, tree, null); } diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index a326b5792be..f915dc1e8c1 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -1124,14 +1124,17 @@ }, "com.yahoo.jdisc.http.ssl.SslContextFactoryProvider": { "superClass": "java.lang.Object", - "interfaces": [], + "interfaces": [ + "java.lang.AutoCloseable" + ], "attributes": [ "public", "interface", "abstract" ], "methods": [ - "public abstract org.eclipse.jetty.util.ssl.SslContextFactory getInstance(java.lang.String, int)" + "public abstract org.eclipse.jetty.util.ssl.SslContextFactory getInstance(java.lang.String, int)", + "public void close()" ], "fields": [] } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java index 37916fd5734..c364116e0af 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java @@ -8,7 +8,7 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; * * @author bjorncs */ -public interface SslContextFactoryProvider { +public interface SslContextFactoryProvider extends AutoCloseable { /** * This method is called once for each SSL connector. @@ -17,4 +17,5 @@ public interface SslContextFactoryProvider { */ SslContextFactory getInstance(String containerId, int port); + @Override default void close() {} } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java index 0bbe6207294..615cd5d46ad 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java @@ -3,7 +3,7 @@ package com.yahoo.jdisc.http.ssl.impl; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; -import com.yahoo.security.tls.ReloadingTlsContext; +import com.yahoo.security.tls.ConfigFileBasedTlsContext; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.TransportSecurityUtils; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -15,21 +15,38 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; */ public class DefaultSslContextFactoryProvider extends AbstractComponent implements SslContextFactoryProvider { - private final TlsContext tlsContext = TransportSecurityUtils.getConfigFile() - .map(configFile -> new ReloadingTlsContext(configFile, TransportSecurityUtils.getInsecureAuthorizationMode())) - .orElse(null); + private final SslContextFactoryProvider instance = TransportSecurityUtils.getConfigFile() + .map(configFile -> (SslContextFactoryProvider) new StaticTlsContextBasedProvider( + new ConfigFileBasedTlsContext(configFile, TransportSecurityUtils.getInsecureAuthorizationMode()))) + .orElseGet(ThrowingSslContextFactoryProvider::new); @Override public SslContextFactory getInstance(String containerId, int port) { - if (tlsContext != null) { - return new TlsContextManagedSslContextFactory(tlsContext); - } else { - throw new UnsupportedOperationException(); - } + return instance.getInstance(containerId, port); } @Override public void deconstruct() { - if (tlsContext != null) tlsContext.close(); + instance.close(); + } + + private static class ThrowingSslContextFactoryProvider implements SslContextFactoryProvider { + @Override + public SslContextFactory getInstance(String containerId, int port) { + throw new UnsupportedOperationException(); + } + } + + private static class StaticTlsContextBasedProvider extends TlsContextBasedProvider { + final TlsContext tlsContext; + + StaticTlsContextBasedProvider(TlsContext tlsContext) { + this.tlsContext = tlsContext; + } + + @Override + protected TlsContext getTlsContext(String containerId, int port) { + return tlsContext; + } } }
\ No newline at end of file diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java new file mode 100644 index 00000000000..e8ae13e48be --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java @@ -0,0 +1,54 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.security.tls.TlsContext; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import java.util.Arrays; + +/** + * A {@link SslContextFactoryProvider} that creates {@link SslContextFactory} instances from {@link TlsContext} instances. + * + * @author bjorncs + */ +public abstract class TlsContextBasedProvider extends AbstractComponent implements SslContextFactoryProvider { + + protected abstract TlsContext getTlsContext(String containerId, int port); + + @Override + public final SslContextFactory getInstance(String containerId, int port) { + TlsContext tlsContext = getTlsContext(containerId, port); + SSLContext sslContext = tlsContext.context(); + SSLParameters parameters = tlsContext.parameters(); + + SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); + sslContextFactory.setSslContext(sslContext); + + sslContextFactory.setNeedClientAuth(parameters.getNeedClientAuth()); + sslContextFactory.setWantClientAuth(parameters.getWantClientAuth()); + + String[] enabledProtocols = parameters.getProtocols(); + sslContextFactory.setIncludeProtocols(enabledProtocols); + String[] supportedProtocols = sslContext.getSupportedSSLParameters().getProtocols(); + sslContextFactory.setExcludeProtocols(createExclusionList(enabledProtocols, supportedProtocols)); + + String[] enabledCiphers = parameters.getCipherSuites(); + String[] supportedCiphers = sslContext.getSupportedSSLParameters().getCipherSuites(); + sslContextFactory.setIncludeCipherSuites(enabledCiphers); + sslContextFactory.setExcludeCipherSuites(createExclusionList(enabledCiphers, supportedCiphers)); + return sslContextFactory; + } + + private static String[] createExclusionList(String[] enabledValues, String[] supportedValues) { + return Arrays.stream(supportedValues) + .filter(supportedValue -> + Arrays.stream(enabledValues) + .noneMatch(enabledValue -> enabledValue.equals(supportedValue))) + .toArray(String[]::new); + } + +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java deleted file mode 100644 index a5652042f9e..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl.impl; - -import com.yahoo.security.tls.TlsContext; -import org.eclipse.jetty.util.ssl.SslContextFactory; - -import javax.net.ssl.SSLEngine; -import java.net.InetSocketAddress; - -/** - * A Jetty {@link SslContextFactory} backed by {@link TlsContext}. - * Overrides methods that are used by Jetty to construct ssl sockets and ssl engines. - * - * @author bjorncs - */ -class TlsContextManagedSslContextFactory extends SslContextFactory.Server { - - private final TlsContext tlsContext; - - TlsContextManagedSslContextFactory(TlsContext tlsContext) { - this.tlsContext = tlsContext; - } - - @Override protected void doStart() { } // Override default behaviour - @Override protected void doStop() { } // Override default behaviour - - @Override - public SSLEngine newSSLEngine() { - return tlsContext.createSslEngine(); - } - - @Override - public SSLEngine newSSLEngine(InetSocketAddress address) { - return tlsContext.createSslEngine(address.getHostString(), address.getPort()); - } - - @Override - public SSLEngine newSSLEngine(String host, int port) { - return tlsContext.createSslEngine(host, port); - } - -} diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java new file mode 100644 index 00000000000..88db5c99de9 --- /dev/null +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java @@ -0,0 +1,70 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.tls.AuthorizationMode; +import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.PeerAuthentication; +import com.yahoo.security.tls.TlsContext; +import com.yahoo.security.tls.policy.AuthorizedPeers; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Set; + +import static com.yahoo.security.KeyAlgorithm.EC; +import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author bjorncs + */ +public class TlsContextBasedProviderTest { + + @Test + public void creates_sslcontextfactory_from_tlscontext() { + TlsContext tlsContext = createTlsContext(); + var provider = new SimpleTlsContextBasedProvider(tlsContext); + SslContextFactory sslContextFactory = provider.getInstance("dummyContainerId", 8080); + assertNotNull(sslContextFactory); + assertArrayEquals(tlsContext.parameters().getCipherSuites(), sslContextFactory.getIncludeCipherSuites()); + } + + private static TlsContext createTlsContext() { + KeyPair keyPair = KeyUtils.generateKeypair(EC); + X509Certificate certificate = X509CertificateBuilder + .fromKeypair( + keyPair, + new X500Principal("CN=dummy"), + Instant.EPOCH, + Instant.EPOCH.plus(100000, ChronoUnit.DAYS), + SHA256_WITH_ECDSA, + BigInteger.ONE) + .build(); + return new DefaultTlsContext( + List.of(certificate), keyPair.getPrivate(), List.of(certificate), new AuthorizedPeers(Set.of()), AuthorizationMode.ENFORCE, PeerAuthentication.NEED); + } + + private static class SimpleTlsContextBasedProvider extends TlsContextBasedProvider { + final TlsContext tlsContext; + + SimpleTlsContextBasedProvider(TlsContext tlsContext) { + this.tlsContext = tlsContext; + } + + @Override + protected TlsContext getTlsContext(String containerId, int port) { + return tlsContext; + } + + } +}
\ No newline at end of file diff --git a/jrt/src/com/yahoo/jrt/CryptoEngine.java b/jrt/src/com/yahoo/jrt/CryptoEngine.java index 81bf10be187..8812264a3f1 100644 --- a/jrt/src/com/yahoo/jrt/CryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/CryptoEngine.java @@ -4,7 +4,7 @@ package com.yahoo.jrt; import com.yahoo.security.tls.AuthorizationMode; import com.yahoo.security.tls.MixedMode; -import com.yahoo.security.tls.ReloadingTlsContext; +import com.yahoo.security.tls.ConfigFileBasedTlsContext; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.TransportSecurityUtils; @@ -24,7 +24,7 @@ public interface CryptoEngine extends AutoCloseable { return new NullCryptoEngine(); } AuthorizationMode mode = TransportSecurityUtils.getInsecureAuthorizationMode(); - TlsContext tlsContext = new ReloadingTlsContext(TransportSecurityUtils.getConfigFile().get(), mode); + TlsContext tlsContext = new ConfigFileBasedTlsContext(TransportSecurityUtils.getConfigFile().get(), mode); TlsCryptoEngine tlsCryptoEngine = new TlsCryptoEngine(tlsContext); MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode(); switch (mixedMode) { diff --git a/jrt/tests/com/yahoo/jrt/CryptoUtils.java b/jrt/tests/com/yahoo/jrt/CryptoUtils.java index 6890fe88da5..e7e4eea568d 100644 --- a/jrt/tests/com/yahoo/jrt/CryptoUtils.java +++ b/jrt/tests/com/yahoo/jrt/CryptoUtils.java @@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.tls.AuthorizationMode; import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.PeerAuthentication; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.policy.AuthorizedPeers; import com.yahoo.security.tls.policy.HostGlobPattern; @@ -48,7 +49,7 @@ class CryptoUtils { Field.CN, new HostGlobPattern("dummy")))))); static TlsContext createTestTlsContext() { - return new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, DefaultTlsContext.ALLOWED_CIPHER_SUITES); + return new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java index 64ede137e8e..017b2c57370 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java @@ -35,12 +35,13 @@ import static java.util.stream.Collectors.toCollection; public class ExternalMetrics { private static final Logger log = Logger.getLogger(ExternalMetrics.class.getName()); + // NOTE: node service id must be kept in sync with the same constant _value_ used in docker-api:Metrics.java + public static final ServiceId VESPA_NODE_SERVICE_ID = toServiceId("vespa.node"); + public static final DimensionId ROLE_DIMENSION = toDimensionId("role"); public static final DimensionId STATE_DIMENSION = toDimensionId("state"); public static final DimensionId ORCHESTRATOR_STATE_DIMENSION = toDimensionId("orchestratorState"); - public static final ServiceId VESPA_NODE_SERVICE_ID = toServiceId("vespa.node"); - private volatile List<MetricsPacket.Builder> metrics = new ArrayList<>(); private final MetricsConsumers consumers; @@ -58,7 +59,6 @@ public class ExternalMetrics { log.log(DEBUG, () -> "Setting new external metrics with " + externalPackets.size() + " metrics packets."); externalPackets.forEach(packet -> { packet.addConsumers(consumers.getAllConsumers()) - .service(VESPA_NODE_SERVICE_ID) .retainMetrics(metricsToRetain()) .applyOutputNames(outputNamesById()); }); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java index e7feab9926d..edb9fba5307 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java @@ -33,7 +33,7 @@ public class RpcConnector extends AbstractComponent { Spec spec = new Spec(config.port()); try { acceptor = supervisor.listen(spec); - log.log(DEBUG, "Listening on " + spec.host() + ":" + spec.port()); + log.log(DEBUG, "Listening on " + spec.host() + ":" + acceptor.port()); } catch (ListenFailedException e) { stop(); log.log(INFO, "Failed listening at " + spec.host() + ":" + spec.port()); @@ -41,6 +41,10 @@ public class RpcConnector extends AbstractComponent { } } + public int port() { + return acceptor.port(); + } + /** * Adds a method. If a method with the same name already exists, it will be replaced. * @param method The method to add. diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java index e441c353292..bc83712ac70 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java @@ -15,6 +15,7 @@ import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.ServiceId; import ai.vespa.metricsproxy.service.DownService; import ai.vespa.metricsproxy.service.DummyService; import ai.vespa.metricsproxy.service.VespaService; @@ -38,6 +39,7 @@ import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -162,6 +164,21 @@ public class MetricsManagerTest { } @Test + public void application_from_extra_metrics_packets_is_used_as_service_in_result_packets() { + final ServiceId serviceId = toServiceId("custom-service"); + metricsManager.setExtraMetrics(ImmutableList.of( + new MetricsPacket.Builder(serviceId) + .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0))))); + + List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); + MetricsPacket extraPacket = null; + for (MetricsPacket packet : packets) { + if (packet.service.equals(serviceId)) extraPacket = packet; + } + assertNotNull(extraPacket); + } + + @Test public void extra_dimensions_are_added_to_metrics_packets_that_do_not_have_those_dimensions() { metricsManager.setExtraMetrics(ImmutableList.of( new MetricsPacket.Builder(toServiceId("foo")) diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/GenericMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/GenericMetricsHandlerTest.java index 29ab8c66694..dc89e5bb9f2 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/GenericMetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/GenericMetricsHandlerTest.java @@ -37,6 +37,7 @@ import java.util.concurrent.Executors; import static ai.vespa.metricsproxy.core.VespaMetrics.INSTANCE_DIMENSION_ID; import static ai.vespa.metricsproxy.http.GenericMetricsHandler.DEFAULT_PUBLIC_CONSUMER_ID; +import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN; import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper; @@ -74,7 +75,7 @@ public class GenericMetricsHandlerTest { public static void setup() { MetricsManager metricsManager = TestUtil.createMetricsManager(vespaServices, getMetricsConsumers(), getApplicationDimensions(), getNodeDimensions()); metricsManager.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")) + new MetricsPacket.Builder(VESPA_NODE_SERVICE_ID) .timestamp(Instant.now().getEpochSecond()) .putMetrics(ImmutableList.of(new Metric(CPU_METRIC, 12.345))))); GenericMetricsHandler handler = new GenericMetricsHandler(Executors.newSingleThreadExecutor(), metricsManager, vespaServices, getMetricsConsumers()); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java index 11c271d46e4..2cce2f66039 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java @@ -8,6 +8,7 @@ import ai.vespa.metricsproxy.core.ConsumersConfig; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.ServiceId; import com.google.common.collect.ImmutableList; import org.junit.Test; @@ -38,15 +39,17 @@ public class ExternalMetricsTest { } @Test - public void service_id_is_set_to_vespa_node_id() { + public void service_id_from_extra_packets_is_not_replaced() { + final ServiceId SERVICE_ID = toServiceId("do-not-replace"); + MetricsConsumers noConsumers = new MetricsConsumers(new ConsumersConfig.Builder().build()); ExternalMetrics externalMetrics = new ExternalMetrics(noConsumers); externalMetrics.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("replace_with_vespa_node_id")))); + new MetricsPacket.Builder(SERVICE_ID))); List<MetricsPacket.Builder> packets = externalMetrics.getMetrics(); assertEquals(1, packets.size()); - assertEquals(VESPA_NODE_SERVICE_ID, packets.get(0).build().service); + assertEquals(SERVICE_ID, packets.get(0).build().service); } @Test diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java index df1ef9e5035..997c02f7768 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java @@ -44,8 +44,6 @@ public class IntegrationTester implements AutoCloseable { static final String SERVICE_1_CONFIG_ID = "container/qrserver.0"; static final String SERVICE_2_CONFIG_ID = "storage/cluster.storage/storage/0"; - private final int httpPort; - private final int rpcPort; private final RpcConnector connector; private final MockHttpServer mockHttpServer; private final VespaServices vespaServices; @@ -54,16 +52,11 @@ public class IntegrationTester implements AutoCloseable { HttpMetricFetcher.CONNECTION_TIMEOUT = 60000; // 60 secs in unit tests } - IntegrationTester(int httpPort, int rpcPort) { - if (httpPort == 0 || rpcPort == 0) { - throw new IllegalArgumentException("http port and rpc port must be defined"); - } - this.httpPort = httpPort; - this.rpcPort = rpcPort; + IntegrationTester() { try { - mockHttpServer = new MockHttpServer(httpPort, null, STATE_PATH); + mockHttpServer = new MockHttpServer(null, STATE_PATH); } catch (IOException e) { - throw new RuntimeException("Unable to start web server on port:" + httpPort); + throw new RuntimeException("Unable to start web server"); } vespaServices = new VespaServices(servicesConfig(), monitoringConfig(), null); @@ -74,7 +67,8 @@ public class IntegrationTester implements AutoCloseable { NodeDimensions nodeDimensions = new NodeDimensions(nodeDimensionsConfig()); connector = new RpcConnector(rpcConnectorConfig()); - RpcServer server = new RpcServer(connector, vespaServices, new MetricsManager(vespaServices, vespaMetrics, externalMetrics, appDimensions, nodeDimensions)); + RpcServer server = new RpcServer(connector, vespaServices, + new MetricsManager(vespaServices, vespaMetrics, externalMetrics, appDimensions, nodeDimensions)); } MockHttpServer httpServer() { @@ -91,7 +85,7 @@ public class IntegrationTester implements AutoCloseable { private RpcConnectorConfig rpcConnectorConfig() { return new RpcConnectorConfig.Builder() - .port(rpcPort) + .port(0) .build(); } @@ -113,8 +107,8 @@ public class IntegrationTester implements AutoCloseable { private VespaServicesConfig servicesConfig() { return new VespaServicesConfig.Builder() - .service(createService(toServiceId("qrserver"), SERVICE_1_CONFIG_ID, httpPort)) - .service(createService(toServiceId("storagenode"), SERVICE_2_CONFIG_ID, httpPort)) + .service(createService(toServiceId("qrserver"), SERVICE_1_CONFIG_ID, httpPort())) + .service(createService(toServiceId("storagenode"), SERVICE_2_CONFIG_ID, httpPort())) .build(); } @@ -140,4 +134,12 @@ public class IntegrationTester implements AutoCloseable { return new NodeDimensionsConfig.Builder().build(); } + public int rpcPort() { + return connector.port(); + } + + public int httpPort() { + return mockHttpServer.port(); + } + } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java index c882de349c8..55e14381183 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java @@ -36,14 +36,9 @@ public class RpcHealthMetricsTest { private static final String WANTED_RPC_RESPONSE = getFileContents("rpc-json-output-check.json").trim(); - // see factory/doc/port-ranges.txt - private static final int httpPort = 18635; - private static final int rpcPort = 18636; - @Test public void expected_response_is_returned() { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { - + try (IntegrationTester tester = new IntegrationTester()) { MockHttpServer mockHttpServer = tester.httpServer(); mockHttpServer.setResponse(HEALTH_OK_RESPONSE); List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); @@ -61,22 +56,22 @@ public class RpcHealthMetricsTest { assertThat("Status should be failed" + h.getMessage(), h.isOk(), is(false)); assertThat(h.getMessage(), is("SOMETHING FAILED")); - String jsonRPCMessage = getHealthMetrics(qrserver.getMonitoringName()); + String jsonRPCMessage = getHealthMetrics(tester, qrserver.getMonitoringName()); assertThat(jsonRPCMessage, is(WANTED_RPC_RESPONSE)); } } @Test public void non_existent_service_name_returns_an_error_message() { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { - String jsonRPCMessage = getHealthMetrics("non-existing service"); + try (IntegrationTester tester = new IntegrationTester()) { + String jsonRPCMessage = getHealthMetrics(tester, "non-existing service"); assertThat(jsonRPCMessage, is("105: No service with name 'non-existing service'")); } } - private String getHealthMetrics(String service) { + private String getHealthMetrics(IntegrationTester tester, String service) { Supervisor supervisor = new Supervisor(new Transport()); - Target target = supervisor.connect(new Spec("localhost", rpcPort)); + Target target = supervisor.connect(new Spec("localhost", tester.rpcPort())); Request req = new Request("getHealthMetricsForYamas"); req.parameters().add(new StringValue(service)); String returnValue; @@ -93,5 +88,4 @@ public class RpcHealthMetricsTest { return returnValue; } - } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java index 6d1b4f3d3b7..d6084e3e03a 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java @@ -17,7 +17,9 @@ import com.yahoo.jrt.Transport; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.util.List; @@ -34,6 +36,8 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author jobergum @@ -41,17 +45,60 @@ import static org.junit.Assert.assertThat; */ public class RpcMetricsTest { - private static final String METRICS_RESPONSE_CCL = - getFileContents("metrics-storage-simple.json").trim(); + private static final String METRICS_RESPONSE = getFileContents("metrics-storage-simple.json").trim(); + private static final String EXTRA_APP = "extra"; - // see factory/doc/port-ranges.txt - private static final int httpPort = 18633; - private static final int rpcPort = 18634; + private static class RpcClient implements AutoCloseable { + private final Supervisor supervisor; + private final Target target; + + RpcClient(int port) { + supervisor = new Supervisor(new Transport()); + target = supervisor.connect(new Spec("localhost", port)); + } + + @Override + public void close() { + target.close(); + supervisor.transport().shutdown().join(); + } + } + + @Test + public void extra_metrics_are_added_to_output() throws Exception { + String extraMetricsPayload = "{\"timestamp\":1557754772,\"application\":\"" + EXTRA_APP + + "\",\"metrics\":{\"foo.count\":3},\"dimensions\":{\"role\":\"extra-role\"}}"; + + try (IntegrationTester tester = new IntegrationTester()) { + try (RpcClient rpcClient = new RpcClient(tester.rpcPort())) { + Request req = new Request("setExtraMetrics"); + req.parameters().add(new StringValue(extraMetricsPayload)); + invoke(req, rpcClient, false); + String allServicesResponse = getMetricsForYamas(ALL_SERVICES, rpcClient).trim(); + + // Verify that application is used as serviceId, and that metric exists. + JSONObject extraMetrics = findExtraMetricsObject(allServicesResponse); + assertThat(extraMetrics.getJSONObject("metrics").getInt("foo.count"), is(3)); + assertThat(extraMetrics.getJSONObject("dimensions").getString("role"), is("extra-role")); + } + } + } + + private JSONObject findExtraMetricsObject(String jsonResponse) throws JSONException { + JSONArray metrics = new JSONObject(jsonResponse).getJSONArray("metrics"); + for (int i = 0; i < metrics.length(); i++) { + JSONObject jsonObject = metrics.getJSONObject(i); + assertTrue(jsonObject.has("application")); + if (jsonObject.getString("application").equals(EXTRA_APP)) return jsonObject; + } + fail("Metrics from setExtraMetrics was missing."); + throw new RuntimeException(); + } @Test public void testGetMetrics() throws Exception { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { - tester.httpServer().setResponse(METRICS_RESPONSE_CCL); + try (IntegrationTester tester = new IntegrationTester()) { + tester.httpServer().setResponse(METRICS_RESPONSE); List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); assertThat("#Services should be 1 for config id " + SERVICE_1_CONFIG_ID, services.size(), is(1)); @@ -66,34 +113,29 @@ public class RpcMetricsTest { Metric m2 = metrics.getMetric("bar.count"); assertNotNull("Did not find expected metric with name 'bar.count'", m2); - // Setup RPC client - Supervisor supervisor = new Supervisor(new Transport()); - Target target = supervisor.connect(new Spec("localhost", rpcPort)); - - verifyMetricsFromRpcRequest(qrserver, target); + try (RpcClient rpcClient = new RpcClient(tester.rpcPort())) { + verifyMetricsFromRpcRequest(qrserver, rpcClient); - services = tester.vespaServices().getInstancesById(SERVICE_2_CONFIG_ID); - assertThat("#Services should be 1 for config id " + SERVICE_2_CONFIG_ID, services.size(), is(1)); + services = tester.vespaServices().getInstancesById(SERVICE_2_CONFIG_ID); + assertThat("#Services should be 1 for config id " + SERVICE_2_CONFIG_ID, services.size(), is(1)); - VespaService storageService = services.get(0); - verfiyMetricsFromServiceObject(storageService); + VespaService storageService = services.get(0); + verfiyMetricsFromServiceObject(storageService); - String metricsById = getMetricsById(storageService.getConfigId(), target); - assertThat(metricsById, is("'storage.cluster.storage.storage.0'.foo_count=1 ")); + String metricsById = getMetricsById(storageService.getConfigId(), rpcClient); + assertThat(metricsById, is("'storage.cluster.storage.storage.0'.foo_count=1 ")); - String jsonResponse = getMetricsForYamas("non-existing", target).trim(); - assertThat(jsonResponse, is("105: No service with name 'non-existing'")); + String jsonResponse = getMetricsForYamas("non-existing", rpcClient).trim(); + assertThat(jsonResponse, is("105: No service with name 'non-existing'")); - verifyMetricsFromRpcRequestForAllServices(target); + verifyMetricsFromRpcRequestForAllServices(rpcClient); - // Shutdown RPC - target.close(); - supervisor.transport().shutdown().join(); + } } } - private static void verifyMetricsFromRpcRequest(VespaService service, Target target) throws JSONException { - String jsonResponse = getMetricsForYamas(service.getMonitoringName(), target).trim(); + private static void verifyMetricsFromRpcRequest(VespaService service, RpcClient client) throws JSONException { + String jsonResponse = getMetricsForYamas(service.getMonitoringName(), client).trim(); JSONArray metrics = new JSONObject(jsonResponse).getJSONArray("metrics"); assertThat("Expected 3 metric messages", metrics.length(), is(3)); for (int i = 0; i < metrics.length() - 1; i++) { // The last "metric message" contains only status code/message @@ -128,18 +170,18 @@ public class RpcMetricsTest { assertThat("Metric foo did not contain correct dimension for key = bar", foo.getDimensions().get(toDimensionId("bar")), is("foo")); } - private void verifyMetricsFromRpcRequestForAllServices(Target target) throws JSONException { + private void verifyMetricsFromRpcRequestForAllServices(RpcClient client) throws JSONException { // Verify that metrics for all services can be retrieved in one request. - String allServicesResponse = getMetricsForYamas(ALL_SERVICES, target).trim(); + String allServicesResponse = getMetricsForYamas(ALL_SERVICES, client).trim(); JSONArray allServicesMetrics = new JSONObject(allServicesResponse).getJSONArray("metrics"); assertThat(allServicesMetrics.length(), is(5)); } @Test - public void testGetAllMetricNames() { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { + public void testGetAllMetricNames() throws Exception { + try (IntegrationTester tester = new IntegrationTester()) { - tester.httpServer().setResponse(METRICS_RESPONSE_CCL); + tester.httpServer().setResponse(METRICS_RESPONSE); List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); assertThat(services.size(), is(1)); @@ -148,52 +190,48 @@ public class RpcMetricsTest { Metric m = metrics.getMetric("foo.count"); assertNotNull("Did not find expected metric with name 'foo.count'", m); - Metric m2 = metrics.getMetric("bar.count"); assertNotNull("Did not find expected metric with name 'bar'", m2); - // Setup RPC - Supervisor supervisor = new Supervisor(new Transport()); - Target target = supervisor.connect(new Spec("localhost", rpcPort)); - - String response = getAllMetricNamesForService(services.get(0).getMonitoringName(), VESPA_CONSUMER_ID, target); - assertThat(response, is("foo.count=ON;output-name=foo_count,bar.count=OFF,")); - - // Shutdown RPC - target.close(); - supervisor.transport().shutdown().join(); + try (RpcClient rpcClient = new RpcClient(tester.rpcPort())) { + String response = getAllMetricNamesForService(services.get(0).getMonitoringName(), VESPA_CONSUMER_ID, rpcClient); + assertThat(response, is("foo.count=ON;output-name=foo_count,bar.count=OFF,")); + } } } - private static String getMetricsForYamas(String service, Target target) { + private static String getMetricsForYamas(String service, RpcClient client) { Request req = new Request("getMetricsForYamas"); req.parameters().add(new StringValue(service)); - return invoke(req, target); + return invoke(req, client, true); } - private String getMetricsById(String service, Target target) { + private String getMetricsById(String service, RpcClient client) { Request req = new Request("getMetricsById"); req.parameters().add(new StringValue(service)); - return invoke(req, target); + return invoke(req, client, true); } - private String getAllMetricNamesForService(String service, ConsumerId consumer, Target target) { + private String getAllMetricNamesForService(String service, ConsumerId consumer, RpcClient client) { Request req = new Request("getAllMetricNamesForService"); req.parameters().add(new StringValue(service)); req.parameters().add(new StringValue(consumer.id)); - return invoke(req, target); + return invoke(req, client, true); } - private static String invoke(Request req, Target target) { + private static String invoke(Request req, RpcClient client, boolean expectReturnValue) { String returnValue; - target.invokeSync(req, 20.0); + client.target.invokeSync(req, 20.0); if (req.checkReturnTypes("s")) { returnValue = req.returnValues().get(0).asString(); - } else { + } else if (expectReturnValue) { System.out.println(req.methodName() + " from rpcserver - Invocation failed " + req.errorCode() + ": " + req.errorMessage()); returnValue = req.errorCode() + ": " + req.errorMessage(); } + else { + return ""; + } return returnValue; } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java index 725501aacaa..5a174412729 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java @@ -22,8 +22,7 @@ import static org.junit.Assert.assertThat; */ public class ContainerServiceTest { - private MockHttpServer service; - private int csPort; + private MockHttpServer httpServer; @BeforeClass public static void init() { @@ -32,10 +31,9 @@ public class ContainerServiceTest { @Before public void setupHTTPServer() { - csPort = 18637; // see factory/doc/port-ranges.txt try { String response = getFileContents("metrics-container-state-multi-chain.json"); - service = new MockHttpServer(csPort, response, METRICS_PATH); + httpServer = new MockHttpServer(response, METRICS_PATH); } catch (Exception e) { e.printStackTrace(); } @@ -44,7 +42,7 @@ public class ContainerServiceTest { @Test public void testMultipleQueryDimensions() throws JSONException { int count = 0; - VespaService service = VespaService.create("service1", "id", csPort); + VespaService service = VespaService.create("service1", "id", httpServer.port()); for (Metric m : service.getMetrics().getMetrics()) { if (m.getName().equals("queries.rate")) { count++; @@ -63,6 +61,6 @@ public class ContainerServiceTest { @After public void shutdown() { - this.service.close(); + this.httpServer.close(); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java index fdf2fae3081..f7802fd04fb 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java @@ -23,23 +23,26 @@ public class MockHttpServer { /** * Mock http server that will return response as body * - * @param port the port to listen to * @param response the response to return along with 200 OK * @param path the file path that the server will accept requests for. E.g /state/v1/metrics */ - public MockHttpServer(int port, String response, String path) throws IOException { + public MockHttpServer(String response, String path) throws IOException { this.response = response; - this.server = HttpServer.create(new InetSocketAddress(port), 10); + this.server = HttpServer.create(new InetSocketAddress(0), 10); this.server.createContext(path, new MyHandler()); this.server.setExecutor(null); // creates a default executor this.server.start(); - System.out.println("Started web server on port " + port); + System.out.println("Started web server on port " + port()); } public synchronized void setResponse(String r) { this.response = r; } + public int port() { + return server.getAddress().getPort(); + } + public void close() { this.server.stop(0); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java index a0c6b5333cc..ac839e595f9 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java @@ -18,8 +18,7 @@ import static org.junit.Assert.assertThat; * @author Unknown */ public class VespaServiceTest { - private MockHttpServer service; - private int csPort; + private MockHttpServer httpServer; private static final String response; static { @@ -29,9 +28,8 @@ public class VespaServiceTest { @Before public void setupHTTPServer() { - csPort = 18632; // see factory/doc/port-ranges.txt try { - service = new MockHttpServer(csPort, response, METRICS_PATH); + httpServer = new MockHttpServer(response, METRICS_PATH); } catch (Exception e) { e.printStackTrace(); } @@ -56,7 +54,7 @@ public class VespaServiceTest { @Test // TODO: Make it possible to test this without running a HTTP server to create the response public void testMetricsFetching() { - VespaService service = VespaService.create("service1", "id", csPort); + VespaService service = VespaService.create("service1", "id", httpServer.port()); Metrics metrics = service.getMetrics(); assertThat(metrics.getMetric("queries.count").getValue().intValue(), is(28)); @@ -70,7 +68,7 @@ public class VespaServiceTest { @After public void shutdown() { - this.service.close(); + this.httpServer.close(); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index 1811fc0c8f0..d515f0d0353 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.component; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; import java.net.URI; import java.util.Collections; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index d8d6b4781c8..91fcdc89da3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -42,8 +42,8 @@ public class DockerOperationsImpl implements DockerOperations { private static final String MANAGER_NAME = "node-admin"; - private static final String IPV6_NPT_PREFIX = "fd00::"; - private static final String IPV4_NPT_PREFIX = "172.17.0.0"; + private static final InetAddress IPV6_NPT_PREFIX = InetAddresses.forString("fd00::"); + private static final InetAddress IPV4_NPT_PREFIX = InetAddresses.forString("172.17.0.0"); private final Docker docker; private final ProcessExecuter processExecuter; @@ -96,16 +96,12 @@ public class DockerOperationsImpl implements DockerOperations { command.withNetworkMode(networking.getDockerNetworkMode()); if (networking == DockerNetworking.NPT) { - InetAddress ipV6Prefix = InetAddresses.forString(IPV6_NPT_PREFIX); - InetAddress ipV6Local = IPAddresses.prefixTranslate(ipV6Address, ipV6Prefix, 8); + InetAddress ipV6Local = IPAddresses.prefixTranslate(ipV6Address, IPV6_NPT_PREFIX, 8); command.withIpAddress(ipV6Local); // IPv4 - Only present for some containers Optional<InetAddress> ipV4Local = ipAddresses.getIPv4Address(context.node().hostname()) - .map(ipV4Address -> { - InetAddress ipV4Prefix = InetAddresses.forString(IPV4_NPT_PREFIX); - return IPAddresses.prefixTranslate(ipV4Address, ipV4Prefix, 2); - }); + .map(ipV4Address -> IPAddresses.prefixTranslate(ipV4Address, IPV4_NPT_PREFIX, 2)); ipV4Local.ifPresent(command::withIpAddress); addEtcHosts(containerData, context.node().hostname(), ipV4Local, ipV6Local); @@ -303,7 +299,6 @@ public class DockerOperationsImpl implements DockerOperations { context.pathInNodeUnderVespaHome("var/maven"), context.pathInNodeUnderVespaHome("var/mediasearch"), // TODO: Remove when vespa-routing is no more context.pathInNodeUnderVespaHome("var/run"), - context.pathInNodeUnderVespaHome("var/scoreboards"), context.pathInNodeUnderVespaHome("var/service"), context.pathInNodeUnderVespaHome("var/share"), context.pathInNodeUnderVespaHome("var/spool"), diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 26e4dcda88e..167ca15bdbf 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -3,17 +3,14 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.yahoo.config.provision.NodeType; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; -import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig; import java.nio.file.Files; import java.nio.file.Path; @@ -22,11 +19,8 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -35,7 +29,6 @@ import java.util.regex.Pattern; import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameMatches; import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.olderThan; -import static com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig.nodeTypeToRole; import static com.yahoo.yolean.Exceptions.uncheck; /** @@ -47,7 +40,6 @@ public class StorageMaintainer { .ofPattern("yyyyMMddHHmmss").withZone(ZoneOffset.UTC); private final Terminal terminal; - private final DockerOperations dockerOperations; private final CoredumpHandler coredumpHandler; private final Path archiveContainerStoragePath; @@ -57,134 +49,12 @@ public class StorageMaintainer { .expireAfterWrite(5, TimeUnit.MINUTES) .build(); - public StorageMaintainer(Terminal terminal, DockerOperations dockerOperations, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath) { + public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath) { this.terminal = terminal; - this.dockerOperations = dockerOperations; this.coredumpHandler = coredumpHandler; this.archiveContainerStoragePath = archiveContainerStoragePath; } - public void writeMetricsConfig(NodeAgentContext context) { - List<SecretAgentCheckConfig> configs = new ArrayList<>(); - Map<String, Object> tags = generateTags(context); - - // host-life - Path hostLifeCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_host_life"); - configs.add(new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath).withTags(tags)); - - // coredumps (except for the done coredumps which is handled by the host) - Path coredumpCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_coredumps"); - configs.add(new SecretAgentCheckConfig("system-coredumps-processing", 300, coredumpCheckPath, - "--application", "system-coredumps-processing", - "--lastmin", "129600", - "--crit", "1", - "--coredir", context.pathInNodeUnderVespaHome("var/crash/processing").toString()) - .withTags(tags)); - - // athenz certificate check - Path athenzCertExpiryCheckPath = context.pathInNodeUnderVespaHome("libexec64/yms/yms_check_athenz_certs"); - configs.add(new SecretAgentCheckConfig("athenz-certificate-expiry", 60, athenzCertExpiryCheckPath, - "--threshold", "20") - .withRunAsUser("root") - .withTags(tags)); - - if (context.nodeType() != NodeType.config) { - // vespa-health - Path vespaHealthCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health"); - configs.add(new SecretAgentCheckConfig("vespa-health", 60, vespaHealthCheckPath, "all") - .withRunAsUser(context.vespaUser()) - .withTags(tags)); - - // vespa - Path vespaCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa"); - SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all"); - vespaSchedule.withRunAsUser(context.vespaUser()); - if (isConfigserverLike(context.nodeType())) { - Map<String, Object> tagsWithoutNameSpace = new LinkedHashMap<>(tags); - tagsWithoutNameSpace.remove("namespace"); - vespaSchedule.withTags(tagsWithoutNameSpace); - } - configs.add(vespaSchedule); - } - - if (context.nodeType() == NodeType.config || context.nodeType() == NodeType.controller) { - - // configserver/controller - Path configServerNewCheckPath = Paths.get("/usr/bin/curl"); - configs.add(new SecretAgentCheckConfig(nodeTypeToRole(context.nodeType()), 60, configServerNewCheckPath, - "-s", "localhost:19071/yamas-metrics") - .withTags(tags)); - - //zkbackupage - Path zkbackupCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py"); - configs.add(new SecretAgentCheckConfig("zkbackupage", 300, zkbackupCheckPath, - "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/zkbackup.stat").toString(), - "-m", "150", - "-a", "config-zkbackupage") - .withTags(tags)); - - String appName = nodeTypeToRole(context.nodeType()) + "-logd"; - Path logdCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/convert-state-metrics-2-yamas.py"); - configs.add(new SecretAgentCheckConfig(appName, 60, logdCheckPath, - appName, "http://localhost:19089/state/v1/metrics") - .withTags(tags)); - } - - if (context.nodeType() == NodeType.proxy) { - //routing-configage - Path routingAgeCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py"); - configs.add(new SecretAgentCheckConfig("routing-configage", 60, routingAgeCheckPath, - "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf.tmp").toString(), - "-m", "1", - "-a", "routing-configage", - "--ignore_file_not_found") - .withTags(tags)); - - //ssl-check - Path sslCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ssl_status"); - configs.add(new SecretAgentCheckConfig("ssl-status", 300, sslCheckPath, - "-e", "localhost", - "-p", "4443", - "-t", "30") - .withTags(tags)); - } - - // Write config and restart yamas-agent - Path yamasAgentFolder = context.pathOnHostFromPathInNode("/etc/yamas-agent"); - configs.forEach(s -> uncheck(() -> s.writeTo(yamasAgentFolder))); - dockerOperations.executeCommandInContainerAsRoot(context, "service", "yamas-agent", "restart"); - } - - private Map<String, Object> generateTags(NodeAgentContext context) { - Map<String, String> tags = new LinkedHashMap<>(); - tags.put("namespace", "Vespa"); - tags.put("role", nodeTypeToRole(context.node().type())); - tags.put("zone", context.zone().getId().value()); - context.node().currentVespaVersion().ifPresent(version -> tags.put("vespaVersion", version.toFullString())); - - if (! isConfigserverLike(context.nodeType())) { - tags.put("state", context.node().state().toString()); - context.node().parentHostname().ifPresent(parent -> tags.put("parentHostname", parent)); - context.node().owner().ifPresent(owner -> { - tags.put("tenantName", owner.tenant()); - tags.put("app", owner.application() + "." + owner.instance()); - tags.put("applicationName", owner.application()); - tags.put("instanceName", owner.instance()); - tags.put("applicationId", owner.tenant() + "." + owner.application() + "." + owner.instance()); - }); - context.node().membership().ifPresent(membership -> { - tags.put("clustertype", membership.clusterType()); - tags.put("clusterid", membership.clusterId()); - }); - } - - return Collections.unmodifiableMap(tags); - } - - private boolean isConfigserverLike(NodeType nodeType) { - return nodeType == NodeType.config || nodeType == NodeType.controller; - } - public Optional<Long> getDiskUsageFor(NodeAgentContext context) { try { Path path = context.pathOnHostFromPathInNode("/"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 550d6e7021e..ce7a99fd841 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -9,7 +9,6 @@ import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; import com.yahoo.vespa.athenz.client.zts.ZtsClient; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index cb10eac9e6c..5d2639d0a77 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -149,7 +149,7 @@ public class NodeAdminImpl implements NodeAdmin { @Override public Duration subsystemFreezeDuration() { if (startOfFreezeConvergence == null) { - return Duration.ofSeconds(0); + return Duration.ZERO; } else { return Duration.between(startOfFreezeConvergence, clock.instant()); } @@ -172,7 +172,7 @@ public class NodeAdminImpl implements NodeAdmin { @Override public void stop() { // Stop all node-agents in parallel, will block until the last NodeAgent is stopped - nodeAgentWithSchedulerByHostname.values().parallelStream().forEach(NodeAgent::stopForRemoval); + nodeAgentWithSchedulerByHostname.values().parallelStream().forEach(NodeAgentWithScheduler::stopForRemoval); } // Set-difference. Returns minuend minus subtrahend. @@ -182,7 +182,7 @@ public class NodeAdminImpl implements NodeAdmin { return result; } - static class NodeAgentWithScheduler implements NodeAgent, NodeAgentScheduler { + static class NodeAgentWithScheduler implements NodeAgentScheduler { private final NodeAgent nodeAgent; private final NodeAgentScheduler nodeAgentScheduler; @@ -191,14 +191,15 @@ public class NodeAdminImpl implements NodeAdmin { this.nodeAgentScheduler = nodeAgentScheduler; } - @Override public void start() { nodeAgent.start(); } - @Override public void stopForHostSuspension() { nodeAgent.stopForHostSuspension(); } - @Override public void stopForRemoval() { nodeAgent.stopForRemoval(); } - @Override public void updateContainerNodeMetrics() { nodeAgent.updateContainerNodeMetrics(); } - @Override public int getAndResetNumberOfUnhandledExceptions() { return nodeAgent.getAndResetNumberOfUnhandledExceptions(); } + void start() { nodeAgent.start(currentContext()); } + void stopForHostSuspension() { nodeAgent.stopForHostSuspension(currentContext()); } + void stopForRemoval() { nodeAgent.stopForRemoval(currentContext()); } + void updateContainerNodeMetrics() { nodeAgent.updateContainerNodeMetrics(currentContext()); } + int getAndResetNumberOfUnhandledExceptions() { return nodeAgent.getAndResetNumberOfUnhandledExceptions(); } @Override public void scheduleTickWith(NodeAgentContext context, Instant at) { nodeAgentScheduler.scheduleTickWith(context, at); } @Override public boolean setFrozen(boolean frozen, Duration timeout) { return nodeAgentScheduler.setFrozen(frozen, timeout); } + @Override public NodeAgentContext currentContext() { return nodeAgentScheduler.currentContext(); } } @FunctionalInterface diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java index de5ee1b69a4..f537884e708 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java @@ -13,24 +13,24 @@ public interface NodeAgent { * Starts the agent. After this method is called, the agent will asynchronously maintain the node, continuously * striving to make the current state equal to the wanted state. */ - void start(); + void start(NodeAgentContext context); /** * Stop the node in anticipation of host suspension, e.g. reboot or docker upgrade. */ - void stopForHostSuspension(); + void stopForHostSuspension(NodeAgentContext context); /** * Signals to the agent that the node is at the end of its lifecycle and no longer needs a managing agent. * Cleans up any resources the agent owns, such as threads, connections etc. Cleanup is synchronous; when this * method returns, no more actions will be taken by the agent. */ - void stopForRemoval(); + void stopForRemoval(NodeAgentContext context); /** * Updates metric receiver with the latest node-agent stats */ - void updateContainerNodeMetrics(); + default void updateContainerNodeMetrics(NodeAgentContext context) {} /** * Returns and resets number of unhandled exceptions diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java index 1fc730a3cb0..65611886f9c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java @@ -13,9 +13,6 @@ public interface NodeAgentContextSupplier { */ NodeAgentContext nextContext() throws InterruptedException; - /** @return the last context returned by {@link #nextContext()} or a default value */ - NodeAgentContext currentContext(); - /** Interrupts the thread(s) currently waiting in {@link #nextContext()} */ void interrupt(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 90eda96d445..77c08133e82 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; -import com.fasterxml.jackson.core.JsonProcessingException; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.zone.ZoneApi; @@ -12,13 +11,8 @@ import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerResources; -import com.yahoo.vespa.hosted.dockerapi.ContainerStats; import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; -import com.yahoo.vespa.hosted.dockerapi.exception.DockerExecTimeoutException; -import com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics; -import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; -import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; @@ -31,11 +25,8 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; @@ -68,16 +59,16 @@ public class NodeAgentImpl implements NodeAgent { private final Optional<CredentialsMaintainer> credentialsMaintainer; private final Optional<AclMaintainer> aclMaintainer; private final Optional<HealthChecker> healthChecker; - private final DoubleFlag containerCpuCap; + private Thread loopThread; + private ContainerState containerState = UNKNOWN; + private NodeSpec lastNode; + private int numberOfUnhandledException = 0; private long currentRebootGeneration = 0; private Optional<Long> currentRestartGeneration = Optional.empty(); - private final Thread loopThread; - - /** * ABSENT means container is definitely absent - A container that was absent will not suddenly appear without * NodeAgent explicitly starting it. @@ -92,10 +83,6 @@ public class NodeAgentImpl implements NodeAgent { UNKNOWN } - private ContainerState containerState = UNKNOWN; - - private NodeSpec lastNode = null; - private CpuUsageReporter lastCpuMetric = new CpuUsageReporter(); // Created in NodeAdminImpl public NodeAgentImpl( @@ -116,11 +103,15 @@ public class NodeAgentImpl implements NodeAgent { this.credentialsMaintainer = credentialsMaintainer; this.aclMaintainer = aclMaintainer; this.healthChecker = healthChecker; + this.containerCpuCap = Flags.CONTAINER_CPU_CAP.bindTo(flagSource); + } - this.containerCpuCap = Flags.CONTAINER_CPU_CAP.bindTo(flagSource) - .with(FetchVector.Dimension.HOSTNAME, contextSupplier.currentContext().node().hostname()); + @Override + public void start(NodeAgentContext initialContext) { + if (loopThread != null) + throw new IllegalStateException("Can not re-start a node agent."); - this.loopThread = new Thread(() -> { + loopThread = new Thread(() -> { while (!terminated.get()) { try { NodeAgentContext context = contextSupplier.nextContext(); @@ -128,19 +119,15 @@ public class NodeAgentImpl implements NodeAgent { } catch (InterruptedException ignored) { } } }); - this.loopThread.setName("tick-" + contextSupplier.currentContext().hostname()); - } - - @Override - public void start() { + loopThread.setName("tick-" + initialContext.hostname()); loopThread.start(); } @Override - public void stopForRemoval() { - if (!terminated.compareAndSet(false, true)) { - throw new RuntimeException("Can not re-stop a node agent."); - } + public void stopForRemoval(NodeAgentContext context) { + if (!terminated.compareAndSet(false, true)) + throw new IllegalStateException("Can not re-stop a node agent."); + contextSupplier.interrupt(); do { @@ -149,7 +136,7 @@ public class NodeAgentImpl implements NodeAgent { } catch (InterruptedException ignored) { } } while (loopThread.isAlive()); - contextSupplier.currentContext().log(logger, "Stopped"); + context.log(logger, "Stopped"); } void startServicesIfNeeded(NodeAgentContext context) { @@ -209,7 +196,6 @@ public class NodeAgentImpl implements NodeAgent { ContainerData containerData = createContainerData(context); dockerOperations.createContainer(context, containerData, getContainerResources(context)); dockerOperations.startContainer(context); - lastCpuMetric = new CpuUsageReporter(); hasStartedServices = true; // Automatically started with the container hasResumedNode = false; @@ -255,8 +241,7 @@ public class NodeAgentImpl implements NodeAgent { } } - private void stopServices() { - NodeAgentContext context = contextSupplier.currentContext(); + private void stopServices(NodeAgentContext context) { context.log(logger, "Stopping services"); if (containerState == ABSENT) return; try { @@ -268,13 +253,11 @@ public class NodeAgentImpl implements NodeAgent { } @Override - public void stopForHostSuspension() { - NodeAgentContext context = contextSupplier.currentContext(); + public void stopForHostSuspension(NodeAgentContext context) { getContainer(context).ifPresent(container -> removeContainer(context, container, "suspending host", true)); } - public void suspend() { - NodeAgentContext context = contextSupplier.currentContext(); + public void suspend(NodeAgentContext context) { context.log(logger, "Suspending services on node"); if (containerState == ABSENT) return; try { @@ -331,9 +314,9 @@ public class NodeAgentImpl implements NodeAgent { try { if (context.node().state() != NodeState.dirty) { - suspend(); + suspend(context); } - stopServices(); + stopServices(context); } catch (Exception e) { context.log(logger, LogLevel.WARNING, "Failed stopping services, ignoring", e); } @@ -365,6 +348,7 @@ public class NodeAgentImpl implements NodeAgent { .map(NodeOwner::asApplicationId) .map(appId -> containerCpuCap.with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm())) .orElse(containerCpuCap) + .with(FetchVector.Dimension.HOSTNAME, context.node().hostname()) .value() * context.node().vcpus(); return ContainerResources.from(cpuCap, context.node().vcpus(), context.node().memoryGb()); @@ -415,12 +399,6 @@ public class NodeAgentImpl implements NodeAgent { currentRestartGeneration.map(current -> current < node.currentRestartGeneration().get()).orElse(false)) currentRestartGeneration = node.currentRestartGeneration(); - // Every time the node spec changes, we should clear the metrics for this container as the dimensions - // will change and we will be reporting duplicate metrics. - if (container.map(c -> c.state.isRunning()).orElse(false)) { - storageMaintainer.writeMetricsConfig(context); - } - lastNode = node; } @@ -513,100 +491,6 @@ public class NodeAgentImpl implements NodeAgent { } } - @SuppressWarnings("unchecked") - public void updateContainerNodeMetrics() { - if (containerState != UNKNOWN) return; - final NodeAgentContext context = contextSupplier.currentContext(); - final NodeSpec node = context.node(); - - Optional<ContainerStats> containerStats = dockerOperations.getContainerStats(context); - if (!containerStats.isPresent()) return; - - Dimensions.Builder dimensionsBuilder = new Dimensions.Builder() - .add("host", context.hostname().value()) - .add("role", SecretAgentCheckConfig.nodeTypeToRole(context.nodeType())) - .add("state", node.state().toString()); - node.parentHostname().ifPresent(parent -> dimensionsBuilder.add("parentHostname", parent)); - node.allowedToBeDown().ifPresent(allowed -> - dimensionsBuilder.add("orchestratorState", allowed ? "ALLOWED_TO_BE_DOWN" : "NO_REMARKS")); - Dimensions dimensions = dimensionsBuilder.build(); - - ContainerStats stats = containerStats.get(); - final String APP = Metrics.APPLICATION_NODE; - final int totalNumCpuCores = stats.getCpuStats().getOnlineCpus(); - final long memoryTotalBytes = stats.getMemoryStats().getLimit(); - final long memoryTotalBytesUsage = stats.getMemoryStats().getUsage(); - final long memoryTotalBytesCache = stats.getMemoryStats().getCache(); - final long diskTotalBytes = (long) (node.diskGb() * BYTES_IN_GB); - final Optional<Long> diskTotalBytesUsed = storageMaintainer.getDiskUsageFor(context); - - lastCpuMetric.updateCpuDeltas(stats.getCpuStats()); - - // Ratio of CPU cores allocated to this container to total number of CPU cores on this host - final double allocatedCpuRatio = node.vcpus() / totalNumCpuCores; - double cpuUsageRatioOfAllocated = lastCpuMetric.getCpuUsageRatio() / allocatedCpuRatio; - double cpuKernelUsageRatioOfAllocated = lastCpuMetric.getCpuKernelUsageRatio() / allocatedCpuRatio; - double cpuThrottledTimeRate = lastCpuMetric.getThrottledTimeRate(); - double cpuThrottledCpuTimeRate = lastCpuMetric.getThrottledCpuTimeRate(); - - long memoryTotalBytesUsed = memoryTotalBytesUsage - memoryTotalBytesCache; - double memoryUsageRatio = (double) memoryTotalBytesUsed / memoryTotalBytes; - double memoryTotalUsageRatio = (double) memoryTotalBytesUsage / memoryTotalBytes; - Optional<Double> diskUsageRatio = diskTotalBytesUsed.map(used -> (double) used / diskTotalBytes); - - List<DimensionMetrics> metrics = new ArrayList<>(); - DimensionMetrics.Builder systemMetricsBuilder = new DimensionMetrics.Builder(APP, dimensions) - .withMetric("mem.limit", memoryTotalBytes) - .withMetric("mem.used", memoryTotalBytesUsed) - .withMetric("mem.util", 100 * memoryUsageRatio) - .withMetric("mem_total.used", memoryTotalBytesUsage) - .withMetric("mem_total.util", 100 * memoryTotalUsageRatio) - .withMetric("cpu.util", 100 * cpuUsageRatioOfAllocated) - .withMetric("cpu.sys.util", 100 * cpuKernelUsageRatioOfAllocated) - .withMetric("cpu.throttled_time.rate", cpuThrottledTimeRate) - .withMetric("cpu.throttled_cpu_time.rate", cpuThrottledCpuTimeRate) - .withMetric("cpu.vcpus", node.vcpus()) - .withMetric("disk.limit", diskTotalBytes); - - diskTotalBytesUsed.ifPresent(diskUsed -> systemMetricsBuilder.withMetric("disk.used", diskUsed)); - diskUsageRatio.ifPresent(diskRatio -> systemMetricsBuilder.withMetric("disk.util", 100 * diskRatio)); - metrics.add(systemMetricsBuilder.build()); - - stats.getNetworks().forEach((interfaceName, interfaceStats) -> { - Dimensions netDims = dimensionsBuilder.add("interface", interfaceName).build(); - DimensionMetrics networkMetrics = new DimensionMetrics.Builder(APP, netDims) - .withMetric("net.in.bytes", interfaceStats.getRxBytes()) - .withMetric("net.in.errors", interfaceStats.getRxErrors()) - .withMetric("net.in.dropped", interfaceStats.getRxDropped()) - .withMetric("net.out.bytes", interfaceStats.getTxBytes()) - .withMetric("net.out.errors", interfaceStats.getTxErrors()) - .withMetric("net.out.dropped", interfaceStats.getTxDropped()) - .build(); - metrics.add(networkMetrics); - }); - - pushMetricsToContainer(context, metrics); - } - - private void pushMetricsToContainer(NodeAgentContext context, List<DimensionMetrics> metrics) { - StringBuilder params = new StringBuilder(); - try { - for (DimensionMetrics dimensionMetrics : metrics) { - params.append(dimensionMetrics.toSecretAgentReport()); - } - String wrappedMetrics = "s:" + params.toString(); - - // Push metrics to the metrics proxy in each container. - // TODO Remove port selection logic when all hosted apps have upgraded to Vespa 7. - int port = context.node().currentVespaVersion().map(version -> version.getMajor() == 6).orElse(false) ? 19091 : 19095; - String[] command = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:" + port, "setExtraMetrics", wrappedMetrics}; - dockerOperations.executeCommandInContainerAsRoot(context, 5L, command); - } catch (JsonProcessingException | DockerExecTimeoutException e) { - context.log(logger, LogLevel.WARNING, "Failed to push metrics to container", e); - } - - } - private Optional<Container> getContainer(NodeAgentContext context) { if (containerState == ABSENT) return Optional.empty(); Optional<Container> container = dockerOperations.getContainer(context); @@ -621,66 +505,6 @@ public class NodeAgentImpl implements NodeAgent { return temp; } - class CpuUsageReporter { - private static final double PERIOD_IN_NANOSECONDS = 1_000d * ContainerResources.CPU_PERIOD_US; - private long containerKernelUsage = 0; - private long totalContainerUsage = 0; - private long totalSystemUsage = 0; - private long throttledTime = 0; - private long throttlingActivePeriods = 0; - private long throttledPeriods = 0; - - private long deltaContainerKernelUsage; - private long deltaContainerUsage; - private long deltaSystemUsage; - private long deltaThrottledTime; - private long deltaThrottlingActivePeriods; - private long deltaThrottledPeriods; - - private void updateCpuDeltas(ContainerStats.CpuStats cpuStats) { - // Do not calculate delta during the first tick - that will result in a metric value that is - // average since container start - if (totalSystemUsage != 0) { - deltaSystemUsage = cpuStats.getSystemCpuUsage() - totalSystemUsage; - deltaContainerUsage = cpuStats.getTotalUsage() - totalContainerUsage; - deltaContainerKernelUsage = cpuStats.getUsageInKernelMode() - containerKernelUsage; - deltaThrottledTime = cpuStats.getThrottledTime() - throttledTime; - deltaThrottlingActivePeriods = cpuStats.getThrottlingActivePeriods() - throttlingActivePeriods; - deltaThrottledPeriods = cpuStats.getThrottledPeriods() - throttledPeriods; - } - - totalSystemUsage = cpuStats.getSystemCpuUsage(); - totalContainerUsage = cpuStats.getTotalUsage(); - containerKernelUsage = cpuStats.getUsageInKernelMode(); - throttledTime = cpuStats.getThrottledTime(); - throttlingActivePeriods = cpuStats.getThrottlingActivePeriods(); - throttledPeriods = cpuStats.getThrottledPeriods(); - } - - /** - * Returns the CPU usage ratio for the docker container that this NodeAgent is managing - * in the time between the last two times updateCpuDeltas() was called. This is calculated - * by dividing the CPU time used by the container with the CPU time used by the entire system. - */ - double getCpuUsageRatio() { - return deltaSystemUsage == 0 ? Double.NaN : (double) deltaContainerUsage / deltaSystemUsage; - } - - double getCpuKernelUsageRatio() { - return deltaSystemUsage == 0 ? Double.NaN : (double) deltaContainerKernelUsage / deltaSystemUsage; - } - - double getThrottledTimeRate() { - return deltaThrottlingActivePeriods == 0 ? Double.NaN : - (double) deltaThrottledPeriods / deltaThrottlingActivePeriods; - } - - double getThrottledCpuTimeRate() { - return deltaThrottlingActivePeriods == 0 ? Double.NaN : - deltaThrottledTime / (PERIOD_IN_NANOSECONDS * deltaThrottlingActivePeriods); - } - } - // TODO: Also skip orchestration if we're downgrading in test/staging // How to implement: // - test/staging: We need to figure out whether we're in test/staging, zone is available in Environment diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java index a5daab8dcfd..956302dcdcc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java @@ -19,4 +19,7 @@ public interface NodeAgentScheduler { * @return True if node agent has converged to the desired state */ boolean setFrozen(boolean frozen, Duration timeout); + + /** @return the last scheduled context or a default value */ + NodeAgentContext currentContext(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java index cc31374669c..5bd3d7800e6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java @@ -19,7 +19,6 @@ import java.util.stream.Collectors; */ @ThreadSafe public class DebugHandlerHelper implements NodeAdminDebugHandler { - private Object monitor = new Object(); private final ConcurrentMap<String, Supplier<Object>> suppliers = new ConcurrentHashMap<>(); public void addThreadSafeSupplier(String name, Supplier<Object> threadSafeSupplier) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java index 5400c19d63e..3bcd806bc85 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java @@ -14,7 +14,7 @@ import java.util.logging.Logger; * @author hakonhall */ public class StoredBoolean { - private static Logger logger = Logger.getLogger(StoredBoolean.class.getName()); + private static final Logger logger = Logger.getLogger(StoredBoolean.class.getName()); private final UnixPath path; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java index 113af76972b..e8c22184406 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java @@ -14,8 +14,8 @@ import java.util.Map; public class Templar { private final String template; - private String prefix = "<%="; - private String suffix = "%>"; + private static final String prefix = "<%="; + private static final String suffix = "%>"; private final Map<String, String> settings = new HashMap<>(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 376fda1d2dc..cf6c6c432f4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -60,7 +60,7 @@ public class UnixPath { public Optional<String> readUtf8FileIfExists() { try { - return Optional.of(new String(Files.readAllBytes(path), StandardCharsets.UTF_8)); + return Optional.of(Files.readString(path)); } catch (NoSuchFileException ignored) { return Optional.empty(); } catch (IOException e) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java index ba1952545a0..85a7c065a86 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.function.Function; @@ -37,7 +36,7 @@ public class Yum { .map(formatter -> "%{" + formatter + "}") .collect(Collectors.joining("\\n")); private static final Function<YumPackageName.Builder, List<Function<String, YumPackageName.Builder>>> - PACKAGE_NAME_BUILDERS_GENERATOR = builder -> Arrays.asList( + PACKAGE_NAME_BUILDERS_GENERATOR = builder -> List.of( builder::setName, builder::setEpoch, builder::setVersion, builder::setRelease, builder::setArchitecture); @@ -183,7 +182,7 @@ public class Yum { return new GenericYumCommand( terminal, yumCommand, - Arrays.asList(packages), + List.of(packages), noopPattern); } @@ -209,9 +208,8 @@ public class Yum { } } - @SuppressWarnings("unchecked") public GenericYumCommand enableRepos(String... repos) { - enabledRepo.addAll(Arrays.asList(repos)); + enabledRepo.addAll(List.of(repos)); return this; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java deleted file mode 100644 index cdf67871a1a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.util; - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileWriter; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * Helper class to generate and write the secret-agent check config files. - * - * @author freva - */ -public class SecretAgentCheckConfig { - private final String id; - private final int interval; - private final Path checkExecutable; - private final String[] arguments; - private String user = "nobody"; - private final Map<String, Object> tags = new LinkedHashMap<>(); - - public SecretAgentCheckConfig(String id, int interval, Path checkExecutable, String... arguments) { - this.id = id; - this.interval = interval; - this.checkExecutable = checkExecutable; - this.arguments = arguments; - } - - public SecretAgentCheckConfig withRunAsUser(String user) { - this.user = user; - return this; - } - - public SecretAgentCheckConfig withTag(String tagKey, Object tagValue) { - tags.put(tagKey, tagValue); - return this; - } - - public SecretAgentCheckConfig withTags(Map<String, Object> tags) { - this.tags.clear(); - this.tags.putAll(tags); - return this; - } - - public void setTags(Map<String, Object> tags) { - this.tags.clear(); - this.tags.putAll(tags); - } - - public void writeTo(Path yamasAgentDirectory) throws IOException { - Files.createDirectories(yamasAgentDirectory); - Path scheduleFilePath = yamasAgentDirectory.resolve(id + ".yaml"); - Files.write(scheduleFilePath, render().getBytes()); - } - - public FileWriter getFileWriterTo(Path destinationPath) { - return new FileWriter(destinationPath, this::render); - } - - public String render() { - StringBuilder stringBuilder = new StringBuilder() - .append("- id: ").append(id).append("\n") - .append(" interval: ").append(interval).append("\n") - .append(" user: ").append(user).append("\n") - .append(" check: ").append(checkExecutable.toFile()).append("\n"); - - if (arguments.length > 0) { - stringBuilder.append(" args:\n"); - for (String arg : arguments) { - stringBuilder.append(" - ").append(arg).append("\n"); - } - } - - if (!tags.isEmpty()) { - stringBuilder.append(" tags:\n"); - tags.forEach((key, value) -> - stringBuilder.append(" ").append(key).append(": ").append(value).append("\n")); - } - - return stringBuilder.toString(); - } - - // TODO: Change role dimension to nodeType? - public static String nodeTypeToRole(NodeType nodeType) { - switch (nodeType) { - case tenant: return "tenants"; - case host: return "docker"; - case proxy: return "routing"; - case proxyhost: return "routinghost"; - case config: return "configserver"; - case confighost: return "configserverhost"; - case controller: return "controller"; - case controllerhost: return "controllerhost"; - default: throw new IllegalArgumentException("Unknown node type " + nodeType); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/package-info.java deleted file mode 100644 index 56cb135e723..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.util; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 57b18606def..d034d3c1cd0 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -2,21 +2,9 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; import com.google.common.collect.ImmutableSet; -import com.yahoo.component.Version; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.After; @@ -35,162 +23,15 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; -import static com.yahoo.yolean.Exceptions.uncheck; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * @author dybis */ @RunWith(Enclosed.class) public class StorageMaintainerTest { - private static final DockerOperations docker = mock(DockerOperations.class); - - public static class SecretAgentCheckTests { - private final StorageMaintainer storageMaintainer = new StorageMaintainer(null, docker, null, null); - - @Test - public void tenant() { - Path path = executeAs(NodeType.tenant); - - assertChecks(path, "athenz-certificate-expiry", "host-life", - "system-coredumps-processing", "vespa", "vespa-health"); - - // All dimensions for vespa metrics should be set by metricsproxy - assertCheckEnds(path.resolve("vespa.yaml"), - " args:\n" + - " - all\n"); - - // For non vespa metrics, we need to set all the dimensions ourselves - assertCheckEnds(path.resolve("host-life.yaml"), - "tags:\n" + - " namespace: Vespa\n" + - " role: tenants\n" + - " zone: prod.us-north-1\n" + - " vespaVersion: 6.305.12\n" + - " state: active\n" + - " parentHostname: host123.test.domain.tld\n" + - " tenantName: tenant\n" + - " app: application.instance\n" + - " applicationName: application\n" + - " instanceName: instance\n" + - " applicationId: tenant.application.instance\n" + - " clustertype: clusterType\n" + - " clusterid: clusterId\n"); - } - - @Test - public void proxy() { - Path path = executeAs(NodeType.proxy); - - assertChecks(path, "athenz-certificate-expiry", "host-life", "routing-configage", - "ssl-status", "system-coredumps-processing", "vespa", "vespa-health"); - - // All dimensions for vespa metrics should be set by the source - assertCheckEnds(path.resolve("vespa.yaml"), - " args:\n" + - " - all\n"); - - // For non vespa metrics, we need to set all the dimensions ourselves - assertCheckEnds(path.resolve("host-life.yaml"), - "tags:\n" + - " namespace: Vespa\n" + - " role: routing\n" + - " zone: prod.us-north-1\n" + - " vespaVersion: 6.305.12\n" + - " state: active\n" + - " parentHostname: host123.test.domain.tld\n" + - " tenantName: tenant\n" + - " app: application.instance\n" + - " applicationName: application\n" + - " instanceName: instance\n" + - " applicationId: tenant.application.instance\n" + - " clustertype: clusterType\n" + - " clusterid: clusterId\n"); - } - - @Test - public void configserver() { - Path path = executeAs(NodeType.config); - - assertChecks(path, "athenz-certificate-expiry", "configserver", "configserver-logd", "host-life", - "system-coredumps-processing", "zkbackupage"); - - assertCheckEnds(path.resolve("configserver.yaml"), - " tags:\n" + - " namespace: Vespa\n" + - " role: configserver\n" + - " zone: prod.us-north-1\n" + - " vespaVersion: 6.305.12\n"); - } - - @Test - public void controller() { - Path path = executeAs(NodeType.controller); - - assertChecks(path, "athenz-certificate-expiry", "controller", "controller-logd", "host-life", - "system-coredumps-processing", "vespa", "vespa-health", "zkbackupage"); - - - // Do not set namespace for vespa metrics. WHY? - assertCheckEnds(path.resolve("vespa.yaml"), - " tags:\n" + - " role: controller\n" + - " zone: prod.us-north-1\n" + - " vespaVersion: 6.305.12\n"); - - assertCheckEnds(path.resolve("controller.yaml"), - " tags:\n" + - " namespace: Vespa\n" + - " role: controller\n" + - " zone: prod.us-north-1\n" + - " vespaVersion: 6.305.12\n"); - } - - private Path executeAs(NodeType nodeType) { - ZoneApi zone = mock(ZoneApi.class); - when(zone.getId()).thenReturn(ZoneId.from(Environment.prod, RegionName.from("us-north-1"))); - - NodeSpec nodeSpec = new NodeSpec.Builder() - .hostname("host123-5.test.domain.tld") - .type(nodeType) - .state(NodeState.active) - .parentHostname("host123.test.domain.tld") - .owner(new NodeOwner("tenant", "application", "instance")) - .membership(new NodeMembership("clusterType", "clusterId", null, 0, false)) - .currentVespaVersion(Version.fromString("6.305.12")) - .flavor("d-2-8-50") - .canonicalFlavor("d-2-8-50") - .build(); - NodeAgentContext context = new NodeAgentContextImpl.Builder(nodeSpec) - .fileSystem(TestFileSystem.create()) - .zone(zone) - .build(); - Path path = context.pathOnHostFromPathInNode("/etc/yamas-agent"); - uncheck(() -> Files.createDirectories(path)); - storageMaintainer.writeMetricsConfig(context); - return path; - } - - private void assertCheckEnds(Path checkPath, String contentsEnd) { - String contents = new UnixPath(checkPath).readUtf8File(); - assertTrue(contents, contents.endsWith(contentsEnd)); - } - - private void assertChecks(Path checksPath, String... checkNames) { - List<String> expectedChecks = Stream.of(checkNames).sorted().collect(Collectors.toList()); - List<String> actualChecks = FileFinder.files(checksPath).stream() - .map(FileFinder.FileAttributes::filename) - .map(filename -> filename.replaceAll("\\.yaml$", "")) - .sorted() - .collect(Collectors.toList()); - assertEquals(expectedChecks, actualChecks); - } - } public static class DiskUsageTests { @@ -198,7 +39,7 @@ public class StorageMaintainerTest { @Test public void testDiskUsed() throws IOException { - StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, docker, null, null); + StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, null, null); FileSystem fileSystem = TestFileSystem.create(); NodeAgentContext context = new NodeAgentContextImpl.Builder("host-1.domain.tld").fileSystem(fileSystem).build(); Files.createDirectories(context.pathOnHostFromPathInNode("/")); @@ -212,7 +53,7 @@ public class StorageMaintainerTest { @Test public void testNonExistingDiskUsed() { - StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, docker, null, null); + StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, null, null); long usedBytes = storageMaintainer.getDiskUsedInBytes(null, Paths.get("/fake/path")); assertEquals(0L, usedBytes); } @@ -244,7 +85,7 @@ public class StorageMaintainerTest { // Archive container-1 - StorageMaintainer storageMaintainer = new StorageMaintainer(null, docker, null, pathToArchiveDir); + StorageMaintainer storageMaintainer = new StorageMaintainer(null, null, pathToArchiveDir); storageMaintainer.archiveNodeStorage(context1); // container-1 should be gone from container-storage diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 46af7e7bafd..c0b032bc4d4 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -9,12 +9,8 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.ContainerResources; -import com.yahoo.vespa.hosted.dockerapi.ContainerStats; import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; -import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; @@ -28,21 +24,12 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import org.junit.Test; import org.mockito.InOrder; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import static com.yahoo.yolean.Exceptions.uncheck; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; @@ -76,7 +63,6 @@ public class NodeAgentImplTest { private final NodeRepository nodeRepository = mock(NodeRepository.class); private final Orchestrator orchestrator = mock(Orchestrator.class); private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); - private final Metrics metrics = new Metrics(); private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); private final HealthChecker healthChecker = mock(HealthChecker.class); private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); @@ -152,7 +138,7 @@ public class NodeAgentImplTest { inOrder.verify(dockerOperations, never()).startServices(eq(context)); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); - nodeAgent.stopForHostSuspension(); + nodeAgent.stopForHostSuspension(context); nodeAgent.doConverge(context); inOrder.verify(dockerOperations, never()).startServices(eq(context)); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); // Expect a resume, but no start services @@ -162,7 +148,7 @@ public class NodeAgentImplTest { inOrder.verify(dockerOperations, never()).startServices(eq(context)); inOrder.verify(dockerOperations, never()).resumeNode(eq(context)); - nodeAgent.stopForHostSuspension(); + nodeAgent.stopForHostSuspension(context); nodeAgent.doConverge(context); inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), any(), any()); inOrder.verify(dockerOperations, times(1)).startContainer(eq(context)); @@ -638,81 +624,6 @@ public class NodeAgentImplTest { } @Test - @SuppressWarnings("unchecked") - public void testGetRelevantMetrics() throws Exception { - String json = Files.readString(Paths.get("src/test/resources/docker.stats.json")); - ContainerStats stats2 = ContainerStats.fromJson(json); - ContainerStats stats1 = ContainerStats.fromJson(json.replace("\"cpu_stats\"", "\"cpu_stats2\"").replace("\"precpu_stats\"", "\"cpu_stats\"")); - - NodeOwner owner = new NodeOwner("tester", "testapp", "testinstance"); - NodeMembership membership = new NodeMembership("clustType", "clustId", "grp", 3, false); - final NodeSpec node = nodeBuilder - .wantedDockerImage(dockerImage) - .currentDockerImage(dockerImage) - .state(NodeState.active) - .currentVespaVersion(vespaVersion) - .owner(owner) - .membership(membership) - .memoryGb(2) - .allowedToBeDown(true) - .parentHostname("parent.host.name.yahoo.com") - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(39625000000L)); - when(dockerOperations.getContainerStats(eq(context))) - .thenReturn(Optional.of(stats1)) - .thenReturn(Optional.of(stats2)); - - List<String> expectedMetrics = Stream.of(0, 1) - .map(i -> Paths.get("src/test/resources/expected.container.system.metrics." + i + ".txt")) - .map(path -> uncheck(() -> Files.readString(path))) - .map(content -> content.replaceAll("\\s", "").replaceAll("\\n", "")) - .collect(Collectors.toList()); - int[] counter = {0}; - - doAnswer(invocation -> { - NodeAgentContext calledContainerName = (NodeAgentContext) invocation.getArguments()[0]; - long calledTimeout = (long) invocation.getArguments()[1]; - String[] calledCommand = new String[invocation.getArguments().length - 2]; - System.arraycopy(invocation.getArguments(), 2, calledCommand, 0, calledCommand.length); - calledCommand[calledCommand.length - 1] = calledCommand[calledCommand.length - 1] - .replaceAll("\"timestamp\":\\d+", "\"timestamp\":0") - .replaceAll("([0-9]+\\.[0-9]{1,3})([0-9]*)", "$1"); // Only keep the first 3 decimals - - assertEquals(context, calledContainerName); - assertEquals(5L, calledTimeout); - String[] expectedCommand = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:19095", - "setExtraMetrics", expectedMetrics.get(counter[0])}; - assertArrayEquals("Ivocation #" + counter[0], expectedCommand, calledCommand); - counter[0]++; - return null; - }).when(dockerOperations).executeCommandInContainerAsRoot(any(), any(), any()); - - nodeAgent.updateContainerNodeMetrics(); - nodeAgent.updateContainerNodeMetrics(); - } - - @Test - public void testGetRelevantMetricsForReadyNode() { - final NodeSpec node = nodeBuilder - .state(NodeState.ready) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false); - - when(dockerOperations.getContainerStats(eq(context))).thenReturn(Optional.empty()); - - nodeAgent.updateContainerNodeMetrics(); - - assertEquals(List.of(), metrics.getDefaultMetrics()); - } - - @Test public void testRunningConfigServer() { final NodeSpec node = nodeBuilder .type(NodeType.config) @@ -747,8 +658,6 @@ public class NodeAgentImplTest { private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) { mockGetContainer(dockerImage, isRunning); - doNothing().when(storageMaintainer).writeMetricsConfig(any()); - return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, Optional.of(credentialsMaintainer), Optional.of(aclMaintainer), Optional.of(healthChecker)); @@ -776,8 +685,6 @@ public class NodeAgentImplTest { } private NodeAgentContext createContext(NodeSpec nodeSpec) { - NodeAgentContext context = new NodeAgentContextImpl.Builder(nodeSpec).build(); - when(contextSupplier.currentContext()).thenReturn(context); - return context; + return new NodeAgentContextImpl.Builder(nodeSpec).build(); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java index 333cb81f9d4..e66b3a7aed2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java @@ -11,7 +11,7 @@ import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; -import java.util.Arrays; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -31,7 +31,7 @@ public class ProcessFactoryImplTest { @Test public void testSpawn() { CommandLine commandLine = mock(CommandLine.class); - when(commandLine.getArguments()).thenReturn(Arrays.asList("program")); + when(commandLine.getArguments()).thenReturn(List.of("program")); when(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()).thenReturn(true); when(commandLine.programName()).thenReturn("program"); Path outputPath; @@ -56,8 +56,8 @@ public class ProcessFactoryImplTest { public void testSpawnWithPersistentOutputFile() { class TemporaryFile implements AutoCloseable { - Path path; - TemporaryFile() { + private final Path path; + private TemporaryFile() { String outputFileName = ProcessFactoryImplTest.class.getSimpleName() + "-temporary-test-file.out"; FileAttribute<Set<PosixFilePermission>> fileAttribute = PosixFilePermissions.asFileAttribute( PosixFilePermissions.fromString("rw-------")); @@ -68,7 +68,7 @@ public class ProcessFactoryImplTest { try (TemporaryFile outputPath = new TemporaryFile()) { CommandLine commandLine = mock(CommandLine.class); - when(commandLine.getArguments()).thenReturn(Arrays.asList("program")); + when(commandLine.getArguments()).thenReturn(List.of("program")); when(commandLine.programName()).thenReturn("program"); when(commandLine.getOutputFile()).thenReturn(Optional.of(outputPath.path)); try (ChildProcess2Impl child = processFactory.spawn(commandLine)) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfigTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfigTest.java deleted file mode 100644 index 30263403757..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfigTest.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.util; - -import com.yahoo.config.provision.NodeType; -import org.junit.Test; - -import java.nio.file.Paths; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -/** - * @author freva - */ -public class SecretAgentCheckConfigTest { - - @Test - public void generateFullSecretAgentScheduleTest() { - SecretAgentCheckConfig scheduleMaker = new SecretAgentCheckConfig("system-checks", 60, - Paths.get("/some/test"), "arg1", "arg2 with space") - .withTag("tenantName", "vespa") - .withTag("applicationName", "canary-docker") - .withTag("instanceName", "default") - .withTag("applicationId", "vespa.canary-docker.default") - .withTag("app", "canary-docker.default") - .withTag("clustertype", "container") - .withTag("clusterid", "canary") - .withTag("vespaVersion", "6.13.37") - .withTag("role", "tenants") - .withTag("flavor", "docker") - .withTag("state", "active") - .withTag("zone", "test.us-west-5"); - - assertEquals( - "- id: system-checks\n" + - " interval: 60\n" + - " user: nobody\n" + - " check: /some/test\n" + - " args:\n" + - " - arg1\n" + - " - arg2 with space\n" + - " tags:\n" + - " tenantName: vespa\n" + - " applicationName: canary-docker\n" + - " instanceName: default\n" + - " applicationId: vespa.canary-docker.default\n" + - " app: canary-docker.default\n" + - " clustertype: container\n" + - " clusterid: canary\n" + - " vespaVersion: 6.13.37\n" + - " role: tenants\n" + - " flavor: docker\n" + - " state: active\n" + - " zone: test.us-west-5\n", scheduleMaker.render()); - } - - @Test - public void generateMinimalSecretAgentScheduleTest() { - SecretAgentCheckConfig scheduleMaker = new SecretAgentCheckConfig("system-checks", 60, - Paths.get("/some/test")); - - assertEquals( - "- id: system-checks\n" + - " interval: 60\n" + - " user: nobody\n" + - " check: /some/test\n", scheduleMaker.render()); - } - - @Test - public void generateSecretAgentScheduleWithDifferentUserTest() { - SecretAgentCheckConfig scheduleMaker = new SecretAgentCheckConfig("system-checks", 60, - Paths.get("/some/test")).withRunAsUser("barfoo"); - - assertEquals( - "- id: system-checks\n" + - " interval: 60\n" + - " user: barfoo\n" + - " check: /some/test\n", scheduleMaker.render()); - } - - @Test - public void supportsAllNodeTypes() { - for (NodeType nodeType : NodeType.values()) { - assertNotNull(SecretAgentCheckConfig.nodeTypeToRole(nodeType)); - } - } - -} diff --git a/node-admin/src/test/resources/docker.stats.json b/node-admin/src/test/resources/docker.stats.json deleted file mode 100644 index 5b42d9a2428..00000000000 --- a/node-admin/src/test/resources/docker.stats.json +++ /dev/null @@ -1,376 +0,0 @@ -{ - "read":"2016-10-05T07:28:17.228361751Z", - "precpu_stats":{ - "cpu_usage":{ - "total_usage":332026268600, - "percpu_usage":[ - 46767331190, - 46637593621, - 36196010351, - 38846420953, - 44237804850, - 35751912062, - 44546685143, - 39042510430 - ], - "usage_in_kernelmode":44040000000, - "usage_in_usermode":158940000000 - }, - "system_cpu_usage":5876874910000000, - "throttling_data":{ - "periods":820694, - "throttled_periods":177731, - "throttled_time":81891944744550 - } - }, - "cpu_stats":{ - "cpu_usage":{ - "total_usage":332131163600, - "percpu_usage":[ - 46774042376, - 46639549207, - 36204341756, - 38879138416, - 44256253747, - 35760081676, - 44567860460, - 39049895962 - ], - "usage_in_kernelmode":44106083850, - "usage_in_usermode":158950000000 - }, - "system_cpu_usage":5876882680000000, - "throttling_data":{ - "periods":821264, - "throttled_periods":178201, - "throttled_time":82181944744550 - } - }, - "memory_stats":{ - "usage":1752707072, - "max_usage":1818116096, - "stats":{ - "active_anon":1326051328, - "active_file":188919808, - "cache":678965248, - "hierarchical_memory_limit":4294967296, - "hierarchical_memsw_limit":8589934592, - "inactive_anon":0, - "inactive_file":237735936, - "mapped_file":62976000, - "pgfault":3102812, - "pgmajfault":1403, - "pgpgin":1691151, - "pgpgout":1263244, - "rss":1326026752, - "rss_huge":0, - "swap":0, - "total_active_anon":1326051328, - "total_active_file":188919808, - "total_cache":426680320, - "total_inactive_anon":0, - "total_inactive_file":237735936, - "total_mapped_file":62976000, - "total_pgfault":3102812, - "total_pgmajfault":1403, - "total_pgpgin":1691151, - "total_pgpgout":1263244, - "total_rss":1326026752, - "total_rss_huge":0, - "total_swap":0, - "total_unevictable":0, - "unevictable":0 - }, - "failcnt":0, - "limit":4294967296 - }, - "blkio_stats":{ - "io_service_bytes_recursive":[ - { - "major":252, - "minor":0, - "op":"Read", - "value":53248 - }, - { - "major":252, - "minor":0, - "op":"Write", - "value":602112 - }, - { - "major":252, - "minor":0, - "op":"Sync", - "value":0 - }, - { - "major":252, - "minor":0, - "op":"Async", - "value":655360 - }, - { - "major":252, - "minor":0, - "op":"Total", - "value":655360 - }, - { - "major":7, - "minor":0, - "op":"Read", - "value":308224 - }, - { - "major":7, - "minor":0, - "op":"Write", - "value":573440 - }, - { - "major":7, - "minor":0, - "op":"Sync", - "value":0 - }, - { - "major":7, - "minor":0, - "op":"Async", - "value":881664 - }, - { - "major":7, - "minor":0, - "op":"Total", - "value":881664 - }, - { - "major":253, - "minor":0, - "op":"Read", - "value":308224 - }, - { - "major":253, - "minor":0, - "op":"Write", - "value":573440 - }, - { - "major":253, - "minor":0, - "op":"Sync", - "value":0 - }, - { - "major":253, - "minor":0, - "op":"Async", - "value":881664 - }, - { - "major":253, - "minor":0, - "op":"Total", - "value":881664 - }, - { - "major":253, - "minor":3, - "op":"Read", - "value":343847936 - }, - { - "major":253, - "minor":3, - "op":"Write", - "value":786432 - }, - { - "major":253, - "minor":3, - "op":"Sync", - "value":131072 - }, - { - "major":253, - "minor":3, - "op":"Async", - "value":344503296 - }, - { - "major":253, - "minor":3, - "op":"Total", - "value":344634368 - } - ], - "io_serviced_recursive":[ - { - "major":252, - "minor":0, - "op":"Read", - "value":13 - }, - { - "major":252, - "minor":0, - "op":"Write", - "value":147 - }, - { - "major":252, - "minor":0, - "op":"Sync", - "value":0 - }, - { - "major":252, - "minor":0, - "op":"Async", - "value":160 - }, - { - "major":252, - "minor":0, - "op":"Total", - "value":160 - }, - { - "major":7, - "minor":0, - "op":"Read", - "value":37 - }, - { - "major":7, - "minor":0, - "op":"Write", - "value":124 - }, - { - "major":7, - "minor":0, - "op":"Sync", - "value":0 - }, - { - "major":7, - "minor":0, - "op":"Async", - "value":161 - }, - { - "major":7, - "minor":0, - "op":"Total", - "value":161 - }, - { - "major":253, - "minor":0, - "op":"Read", - "value":37 - }, - { - "major":253, - "minor":0, - "op":"Write", - "value":124 - }, - { - "major":253, - "minor":0, - "op":"Sync", - "value":0 - }, - { - "major":253, - "minor":0, - "op":"Async", - "value":161 - }, - { - "major":253, - "minor":0, - "op":"Total", - "value":161 - }, - { - "major":253, - "minor":3, - "op":"Read", - "value":11812 - }, - { - "major":253, - "minor":3, - "op":"Write", - "value":142 - }, - { - "major":253, - "minor":3, - "op":"Sync", - "value":2 - }, - { - "major":253, - "minor":3, - "op":"Async", - "value":11952 - }, - { - "major":253, - "minor":3, - "op":"Total", - "value":11954 - } - ], - "io_queue_recursive":[ - - ], - "io_service_time_recursive":[ - - ], - "io_wait_time_recursive":[ - - ], - "io_merged_recursive":[ - - ], - "io_time_recursive":[ - - ], - "sectors_recursive":[ - - ] - }, - "pids_stats":{ - - }, - "networks":{ - "eth0":{ - "rx_bytes":19499270, - "rx_packets":58913, - "rx_errors":55, - "rx_dropped":4, - "tx_bytes":20303455, - "tx_packets":62319, - "tx_errors":3, - "tx_dropped":13 - }, - "eth1":{ - "rx_bytes":3245766, - "rx_packets":23462, - "rx_errors":0, - "rx_dropped":0, - "tx_bytes":54246745, - "tx_packets":34562, - "tx_errors":0, - "tx_dropped":0 - } - } -}
\ No newline at end of file diff --git a/node-admin/src/test/resources/expected.container.system.metrics.0.txt b/node-admin/src/test/resources/expected.container.system.metrics.0.txt deleted file mode 100644 index ea6036ce2ea..00000000000 --- a/node-admin/src/test/resources/expected.container.system.metrics.0.txt +++ /dev/null @@ -1,78 +0,0 @@ -s: -{ - "application": "vespa.node", - "dimensions": { - "host": "host1.test.yahoo.com", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "parentHostname": "parent.host.name.yahoo.com", - "role": "tenants", - "state": "active" - }, - "metrics": { - "cpu.vcpus": 2.0, - "disk.limit": 250000000000, - "disk.used": 39625000000, - "disk.util": 15.85, - "mem.limit": 4294967296, - "mem.used": 1073741824, - "mem.util": 25.0, - "mem_total.used": 1752707072, - "mem_total.util": 40.808 - }, - "routing": { - "yamas": { - "namespaces": ["Vespa"] - } - }, - "timestamp": 0 -} -{ - "application": "vespa.node", - "dimensions": { - "host": "host1.test.yahoo.com", - "interface": "eth0", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "parentHostname": "parent.host.name.yahoo.com", - "role": "tenants", - "state": "active" - }, - "metrics": { - "net.in.bytes": 19499270, - "net.in.dropped": 4, - "net.in.errors": 55, - "net.out.bytes": 20303455, - "net.out.dropped": 13, - "net.out.errors": 3 - }, - "routing": { - "yamas": { - "namespaces": ["Vespa"] - } - }, - "timestamp": 0 -} -{ - "application": "vespa.node", - "dimensions": { - "host": "host1.test.yahoo.com", - "interface": "eth1", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "parentHostname": "parent.host.name.yahoo.com", - "role": "tenants", - "state": "active" - }, - "metrics": { - "net.in.bytes": 3245766, - "net.in.dropped": 0, - "net.in.errors": 0, - "net.out.bytes": 54246745, - "net.out.dropped": 0, - "net.out.errors": 0 - }, - "routing": { - "yamas": { - "namespaces": ["Vespa"] - } - }, - "timestamp": 0 -}
\ No newline at end of file diff --git a/node-admin/src/test/resources/expected.container.system.metrics.1.txt b/node-admin/src/test/resources/expected.container.system.metrics.1.txt deleted file mode 100644 index 54d4d36c7d0..00000000000 --- a/node-admin/src/test/resources/expected.container.system.metrics.1.txt +++ /dev/null @@ -1,82 +0,0 @@ -s: -{ - "application": "vespa.node", - "dimensions": { - "host": "host1.test.yahoo.com", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "parentHostname": "parent.host.name.yahoo.com", - "role": "tenants", - "state": "active" - }, - "metrics": { - "cpu.sys.util": 3.402, - "cpu.throttled_cpu_time.rate": 5.087, - "cpu.throttled_time.rate": 0.824, - "cpu.util": 5.4, - "cpu.vcpus": 2.0, - "disk.limit": 250000000000, - "disk.used": 39625000000, - "disk.util": 15.85, - "mem.limit": 4294967296, - "mem.used": 1073741824, - "mem.util": 25.0, - "mem_total.used": 1752707072, - "mem_total.util": 40.808 - }, - "routing": { - "yamas": { - "namespaces": ["Vespa"] - } - }, - "timestamp": 0 -} -{ - "application": "vespa.node", - "dimensions": { - "host": "host1.test.yahoo.com", - "interface": "eth0", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "parentHostname": "parent.host.name.yahoo.com", - "role": "tenants", - "state": "active" - }, - "metrics": { - "net.in.bytes": 19499270, - "net.in.dropped": 4, - "net.in.errors": 55, - "net.out.bytes": 20303455, - "net.out.dropped": 13, - "net.out.errors": 3 - }, - "routing": { - "yamas": { - "namespaces": ["Vespa"] - } - }, - "timestamp": 0 -} -{ - "application": "vespa.node", - "dimensions": { - "host": "host1.test.yahoo.com", - "interface": "eth1", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "parentHostname": "parent.host.name.yahoo.com", - "role": "tenants", - "state": "active" - }, - "metrics": { - "net.in.bytes": 3245766, - "net.in.dropped": 0, - "net.in.errors": 0, - "net.out.bytes": 54246745, - "net.out.dropped": 0, - "net.out.errors": 0 - }, - "routing": { - "yamas": { - "namespaces": ["Vespa"] - } - }, - "timestamp": 0 -}
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index d7f41c4d8e2..438732ad4a8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -41,8 +40,7 @@ import java.util.stream.Collectors; * to failed due to some undetected hardware failure will end up being failed again. * When that has happened enough they will not be recycled. * <p> - * The Chef recipe running locally on the node may set hardwareFailureDescription to avoid the node - * being automatically recycled in cases where an error has been positively detected. + * Nodes with detected hardware issues will not be recycled. * * @author bratseth * @author mpolden diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index e0ec4a6fd06..7e0fcdc0ccc 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -44,7 +44,7 @@ vespalib::string denseSpec("tensor(x[2],y[3])"); Tensor::UP createTensor(const TensorSpec &spec) { auto value = DefaultTensorEngine::ref().from_spec(spec); if (value->is_double()) { - return Tensor::UP(new DenseTensor(ValueType::double_type(), {value->as_double()})); + return Tensor::UP(new DenseTensor<double>(ValueType::double_type(), {value->as_double()})); } Tensor *tensor = dynamic_cast<Tensor*>(value.get()); ASSERT_TRUE(tensor != nullptr); diff --git a/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp b/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp index 6ef680e0505..b7fb3d2b6a1 100644 --- a/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp +++ b/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp @@ -197,25 +197,25 @@ TEST_F("prepareSharedState emits double vector for double imported attribute", A TEST_F("prepareSharedState handles tensor as float from tensor for double imported attribute", ArrayFixture) { f.setup_float_mappings(BasicType::DOUBLE); - vespalib::tensor::DenseTensor tensor(vespalib::eval::ValueType::from_spec("tensor<float>(x[3])"), {10.1, 20.2, 30.3}); + vespalib::tensor::DenseTensor<float> tensor(vespalib::eval::ValueType::from_spec("tensor<float>(x[3])"), {10.1, 20.2, 30.3}); f.template check_prepare_state_output(tensor, dotproduct::ArrayParam<double>({10.1, 20.2, 30.3})); } TEST_F("prepareSharedState handles tensor as double from tensor for double imported attribute", ArrayFixture) { f.setup_float_mappings(BasicType::DOUBLE); - vespalib::tensor::DenseTensor tensor(vespalib::eval::ValueType::from_spec("tensor(x[3])"), {10.1, 20.2, 30.3}); + vespalib::tensor::DenseTensor<double> tensor(vespalib::eval::ValueType::from_spec("tensor(x[3])"), {10.1, 20.2, 30.3}); f.template check_prepare_state_output(tensor, dotproduct::ArrayParam<double>({10.1, 20.2, 30.3})); } TEST_F("prepareSharedState handles tensor as float from tensor for float imported attribute", ArrayFixture) { f.setup_float_mappings(BasicType::FLOAT); - vespalib::tensor::DenseTensor tensor(vespalib::eval::ValueType::from_spec("tensor<float>(x[3])"), {10.1, 20.2, 30.3}); + vespalib::tensor::DenseTensor<float> tensor(vespalib::eval::ValueType::from_spec("tensor<float>(x[3])"), {10.1, 20.2, 30.3}); f.template check_prepare_state_output(tensor, dotproduct::ArrayParam<float>({10.1, 20.2, 30.3})); } TEST_F("prepareSharedState handles tensor as double from tensor for float imported attribute", ArrayFixture) { f.setup_float_mappings(BasicType::FLOAT); - vespalib::tensor::DenseTensor tensor(vespalib::eval::ValueType::from_spec("tensor(x[3])"), {10.1, 20.2, 30.3}); + vespalib::tensor::DenseTensor<double> tensor(vespalib::eval::ValueType::from_spec("tensor(x[3])"), {10.1, 20.2, 30.3}); f.template check_prepare_state_output(tensor, dotproduct::ArrayParam<float>({10.1, 20.2, 30.3})); } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index 205c686df81..f741002ea5e 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -10,7 +10,6 @@ using search::datastore::Handle; using vespalib::tensor::Tensor; -using vespalib::tensor::DenseTensor; using vespalib::tensor::DenseTensorView; using vespalib::tensor::MutableDenseTensorView; using vespalib::eval::ValueType; @@ -41,8 +40,7 @@ DenseTensorStore::TensorSizeCalc::TensorSizeCalc(const ValueType &type) size_t DenseTensorStore::TensorSizeCalc::arraySize() const { - size_t tensorSize = _numBoundCells * _cellSize + - _numUnboundDims * sizeof(uint32_t); + size_t tensorSize = (_numBoundCells * _cellSize) + (_numUnboundDims * sizeof(uint32_t)); return DenseTensorStore::BufferType::align(tensorSize, DENSE_TENSOR_ALIGNMENT); } @@ -185,16 +183,16 @@ void makeConcreteType(MutableDenseTensorView &tensor, std::unique_ptr<Tensor> DenseTensorStore::getTensor(EntryRef ref) const { - using CellsRef = DenseTensorView::CellsRef; if (!ref.valid()) { return std::unique_ptr<Tensor>(); } auto raw = getRawBuffer(ref); size_t numCells = getNumCells(raw); + vespalib::tensor::TypedCells cells_ref(raw, _type.cell_type(), numCells); if (_tensorSizeCalc._numUnboundDims == 0) { - return std::make_unique<DenseTensorView>(_type, CellsRef(static_cast<const double *>(raw), numCells)); + return std::make_unique<DenseTensorView>(_type, cells_ref); } else { - auto result = std::make_unique<MutableDenseTensorView>(_type, CellsRef(static_cast<const double *>(raw), numCells)); + auto result = std::make_unique<MutableDenseTensorView>(_type, cells_ref); makeConcreteType(*result, raw, _tensorSizeCalc._numUnboundDims); return result; } @@ -204,14 +202,16 @@ void DenseTensorStore::getTensor(EntryRef ref, MutableDenseTensorView &tensor) const { if (!ref.valid()) { - tensor.setCells(DenseTensorView::CellsRef(&_emptyCells[0], _emptyCells.size())); + vespalib::tensor::TypedCells cells_ref(&_emptyCells[0], _type.cell_type(), _emptyCells.size()); + tensor.setCells(cells_ref); if (_tensorSizeCalc._numUnboundDims > 0) { tensor.setUnboundDimensionsForEmptyTensor(); } } else { auto raw = getRawBuffer(ref); size_t numCells = getNumCells(raw); - tensor.setCells(DenseTensorView::CellsRef(static_cast<const double *>(raw), numCells)); + vespalib::tensor::TypedCells cells_ref(raw, _type.cell_type(), numCells); + tensor.setCells(cells_ref); if (_tensorSizeCalc._numUnboundDims > 0) { makeConcreteType(tensor, raw, _tensorSizeCalc._numUnboundDims); } @@ -268,11 +268,11 @@ template <class TensorType> TensorStore::EntryRef DenseTensorStore::setDenseTensor(const TensorType &tensor) { - size_t numCells = tensor.cellsRef().size(); + size_t numCells = tensor.cellsRef().size; checkMatchingType(_type, tensor.type(), numCells); auto raw = allocRawBuffer(numCells); setDenseTensorUnboundDimSizes(raw.data, _type, _tensorSizeCalc._numUnboundDims, tensor.type()); - memcpy(raw.data, &tensor.cellsRef()[0], numCells * _tensorSizeCalc._cellSize); + memcpy(raw.data, tensor.cellsRef().data, numCells * _tensorSizeCalc._cellSize); return raw.ref; } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 79b02b09cba..7dd666ac74f 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -4,7 +4,7 @@ #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/tensor_data_type.h> #include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/tensor/dense/dense_tensor.h> +#include <vespa/eval/tensor/dense/typed_dense_tensor_builder.h> #include <vespa/eval/tensor/sparse/sparse_tensor.h> #include <vespa/eval/tensor/wrapped_simple_tensor.h> #include <vespa/vespalib/util/rcuvector.hpp> @@ -12,7 +12,8 @@ using vespalib::eval::SimpleTensor; using vespalib::eval::ValueType; using vespalib::tensor::Tensor; -using vespalib::tensor::DenseTensor; +using vespalib::tensor::TypedDenseTensorBuilder; +using vespalib::tensor::dispatch_0; using vespalib::tensor::SparseTensor; using vespalib::tensor::WrappedSimpleTensor; using document::TensorDataType; @@ -44,17 +45,21 @@ createEmptyTensorType(const ValueType &type) return ValueType::tensor_type(std::move(list)); } +struct CallMakeEmptyTensor { + template <typename CT> + static Tensor::UP call(const ValueType &type) { + TypedDenseTensorBuilder<CT> builder(type); + return builder.build(); + } +}; + Tensor::UP createEmptyTensor(const ValueType &type) { if (type.is_sparse()) { return std::make_unique<SparseTensor>(type, SparseTensor::Cells()); } else if (type.is_dense()) { - size_t size = 1; - for (const auto &dimension : type.dimensions()) { - size *= dimension.size; - } - return std::make_unique<DenseTensor>(type, DenseTensor::Cells(size)); + return dispatch_0<CallMakeEmptyTensor>(type.cell_type(), type); } else { return std::make_unique<WrappedSimpleTensor>(std::make_unique<SimpleTensor>(type, SimpleTensor::Cells())); } diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java index ae18700246c..367d7b9dd83 100644 --- a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java +++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java @@ -51,17 +51,15 @@ public class Main { Map<OutputVariable, String> outputVariables = new TreeMap<>(); Optional<TransportSecurityOptions> options = TransportSecurityUtils.getOptions(envVars); - if (options.isPresent()) { + MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode(envVars); + if (options.isPresent() && mixedMode != MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER) { outputVariables.put(OutputVariable.TLS_ENABLED, "1"); options.get().getCaCertificatesFile() .ifPresent(caCertFile -> outputVariables.put(OutputVariable.CA_CERTIFICATE, caCertFile.toString())); - MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode(envVars); - if (mixedMode != MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER) { - options.get().getCertificatesFile() - .ifPresent(certificateFile -> outputVariables.put(OutputVariable.CERTIFICATE, certificateFile.toString())); - options.get().getPrivateKeyFile() - .ifPresent(privateKeyFile -> outputVariables.put(OutputVariable.PRIVATE_KEY, privateKeyFile.toString())); - } + options.get().getCertificatesFile() + .ifPresent(certificateFile -> outputVariables.put(OutputVariable.CERTIFICATE, certificateFile.toString())); + options.get().getPrivateKeyFile() + .ifPresent(privateKeyFile -> outputVariables.put(OutputVariable.PRIVATE_KEY, privateKeyFile.toString())); } shell.writeOutputVariables(stdOut, outputVariables); EnumSet<OutputVariable> unusedVariables = outputVariables.isEmpty() diff --git a/security-tools/src/main/sh/vespa-curl-wrapper b/security-tools/src/main/sh/vespa-curl-wrapper index 7c2f31d7719..da857984c01 100755 --- a/security-tools/src/main/sh/vespa-curl-wrapper +++ b/security-tools/src/main/sh/vespa-curl-wrapper @@ -6,26 +6,23 @@ set -e -. $(vespa-security-env) +eval $(vespa-security-env) -CURL_PARAMETERS=$1 -CONFIGSERVER_URI_WITHOUT_SCHEME=$2 +CURL_PARAMETERS=("$@") if [ -n "${VESPA_TLS_ENABLED}" ] then - CONFIGSERVER_URI="https://${CONFIGSERVER_URI_WITHOUT_SCHEME}" -else - CONFIGSERVER_URI="http://${CONFIGSERVER_URI_WITHOUT_SCHEME}" + CURL_PARAMETERS=("${CURL_PARAMETERS[@]/http:/https:}") fi if [ -n "${VESPA_TLS_CA_CERT}" ] then - CURL_PARAMETERS="--cacert \"${VESPA_TLS_CA_CERT}\" ${CURL_PARAMETERS}" + CURL_PARAMETERS=("--cacert" "${VESPA_TLS_CA_CERT}" "${CURL_PARAMETERS[@]}") fi if [[ -n "${VESPA_TLS_CERT}" && -n "${VESPA_TLS_PRIVATE_KEY}" ]] then - CURL_PARAMETERS="--cert \"${VESPA_TLS_CERT}\" --key \"${VESPA_TLS_PRIVATE_KEY}\" ${CURL_PARAMETERS}" + CURL_PARAMETERS=("--cert" "${VESPA_TLS_CERT}" "--key" "${VESPA_TLS_PRIVATE_KEY}" "${CURL_PARAMETERS[@]}") fi -curl ${CURL_PARAMETERS} "${CONFIGSERVER_URI}" +curl "${CURL_PARAMETERS[@]}" diff --git a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java index 0ef179f775e..4f8919cdd5e 100644 --- a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java @@ -33,6 +33,7 @@ public class SslContextBuilder { private char[] keyStorePassword; private TrustManagerFactory trustManagerFactory = TrustManagerUtils::createDefaultX509TrustManager; private KeyManagerFactory keyManagerFactory = KeyManagerUtils::createDefaultX509KeyManager; + private X509ExtendedKeyManager keyManager; public SslContextBuilder() {} @@ -110,11 +111,23 @@ public class SslContextBuilder { return this; } + /** + * Note: Callee is responsible for configuring the key manager. + * Any keystore configured by {@link #withKeyStore(KeyStore, char[])} or the other overloads will be ignored. + */ + public SslContextBuilder withKeyManager(X509ExtendedKeyManager keyManager) { + this.keyManager = keyManager; + return this; + } + public SSLContext build() { try { SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); TrustManager[] trustManagers = new TrustManager[] { trustManagerFactory.createTrustManager(trustStoreSupplier.get()) }; - KeyManager[] keyManagers = new KeyManager[] { keyManagerFactory.createKeyManager(keyStoreSupplier.get(), keyStorePassword) }; + X509ExtendedKeyManager keyManager = this.keyManager != null + ? this.keyManager + : keyManagerFactory.createKeyManager(keyStoreSupplier.get(), keyStorePassword); + KeyManager[] keyManagers = new KeyManager[] {keyManager}; sslContext.init(keyManagers, trustManagers, null); return sslContext; } catch (GeneralSecurityException e) { diff --git a/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java b/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java index 0dae185995c..faf6ecb4348 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java @@ -31,6 +31,8 @@ import java.util.logging.Logger; */ public class AutoReloadingX509KeyManager extends X509ExtendedKeyManager implements AutoCloseable { + public static final String CERTIFICATE_ALIAS = "default"; + private static final Duration UPDATE_PERIOD = Duration.ofHours(1); private static final Logger log = Logger.getLogger(AutoReloadingX509KeyManager.class.getName()); @@ -61,7 +63,7 @@ public class AutoReloadingX509KeyManager extends X509ExtendedKeyManager implemen try { return KeyStoreBuilder.withType(KeyStoreType.PKCS12) .withKeyEntry( - "default", + CERTIFICATE_ALIAS, KeyUtils.fromPemEncodedPrivateKey(Files.readString(privateKey)), X509CertificateUtils.certificateListFromPem(Files.readString(certificateChain))) .build(); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ReloadingTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java index 16f66f91da6..3b9158cf9b1 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ReloadingTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java @@ -20,6 +20,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyStore; import java.time.Duration; +import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -29,20 +31,21 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * A {@link TlsContext} that regularly reloads the credentials referred to from the transport security options file. + * A {@link TlsContext} that uses the tls configuration specified in the transport security options file. + * The credentials are regularly reloaded to support short-lived certificates. * * @author bjorncs */ -public class ReloadingTlsContext implements TlsContext { +public class ConfigFileBasedTlsContext implements TlsContext { private static final Duration UPDATE_PERIOD = Duration.ofHours(1); - private static final Logger log = Logger.getLogger(ReloadingTlsContext.class.getName()); + private static final Logger log = Logger.getLogger(ConfigFileBasedTlsContext.class.getName()); private final TlsContext tlsContext; private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ReloaderThreadFactory()); - public ReloadingTlsContext(Path tlsOptionsConfigFile, AuthorizationMode mode) { + public ConfigFileBasedTlsContext(Path tlsOptionsConfigFile, AuthorizationMode mode) { TransportSecurityOptions options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); MutableX509TrustManager trustManager = new MutableX509TrustManager(); MutableX509KeyManager keyManager = new MutableX509KeyManager(); @@ -99,13 +102,15 @@ public class ReloadingTlsContext implements TlsContext { MutableX509TrustManager mutableTrustManager, MutableX509KeyManager mutableKeyManager) { SSLContext sslContext = new SslContextBuilder() - .withKeyManagerFactory((ignoredKeystore, ignoredPassword) -> mutableKeyManager) + .withKeyManager(mutableKeyManager) .withTrustManagerFactory( ignoredTruststore -> options.getAuthorizedPeers() .map(authorizedPeers -> (X509ExtendedTrustManager) new PeerAuthorizerTrustManager(authorizedPeers, mode, mutableTrustManager)) .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(Set.of()), AuthorizationMode.DISABLE, mutableTrustManager))) .build(); - return new DefaultTlsContext(sslContext, options.getAcceptedCiphers()); + List<String> acceptedCiphers = options.getAcceptedCiphers(); + Set<String> ciphers = acceptedCiphers.isEmpty() ? TlsContext.ALLOWED_CIPHER_SUITES : new HashSet<>(acceptedCiphers); + return new DefaultTlsContext(sslContext, ciphers, PeerAuthentication.NEED); } // Wrapped methods from TlsContext diff --git a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java index e74ad49b2f5..572461c6cdd 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java @@ -23,47 +23,41 @@ import java.util.logging.Logger; */ public class DefaultTlsContext implements TlsContext { - public static final List<String> ALLOWED_CIPHER_SUITES = Arrays.asList( - "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "TLS_AES_128_GCM_SHA256", // TLSv1.3 - "TLS_AES_256_GCM_SHA384", // TLSv1.3 - "TLS_CHACHA20_POLY1305_SHA256"); // TLSv1.3 - - public static final List<String> ALLOWED_PROTOCOLS = List.of("TLSv1.2"); // TODO Enable TLSv1.3 - private static final Logger log = Logger.getLogger(DefaultTlsContext.class.getName()); private final SSLContext sslContext; private final String[] validCiphers; private final String[] validProtocols; + private final PeerAuthentication peerAuthentication; public DefaultTlsContext(List<X509Certificate> certificates, PrivateKey privateKey, List<X509Certificate> caCertificates, AuthorizedPeers authorizedPeers, AuthorizationMode mode, - List<String> acceptedCiphers) { - this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode), - acceptedCiphers); + PeerAuthentication peerAuthentication) { + this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode), peerAuthentication); + } + + public DefaultTlsContext(SSLContext sslContext, PeerAuthentication peerAuthentication) { + this(sslContext, TlsContext.ALLOWED_CIPHER_SUITES, peerAuthentication); } + public DefaultTlsContext(SSLContext sslContext) { + this(sslContext, TlsContext.ALLOWED_CIPHER_SUITES, PeerAuthentication.NEED); + } - public DefaultTlsContext(SSLContext sslContext, List<String> acceptedCiphers) { + DefaultTlsContext(SSLContext sslContext, Set<String> acceptedCiphers, PeerAuthentication peerAuthentication) { this.sslContext = sslContext; + this.peerAuthentication = peerAuthentication; this.validCiphers = getAllowedCiphers(sslContext, acceptedCiphers); this.validProtocols = getAllowedProtocols(sslContext); } - - private static String[] getAllowedCiphers(SSLContext sslContext, List<String> acceptedCiphers) { + private static String[] getAllowedCiphers(SSLContext sslContext, Set<String> acceptedCiphers) { String[] supportedCipherSuites = sslContext.getSupportedSSLParameters().getCipherSuites(); String[] validCipherSuites = Arrays.stream(supportedCipherSuites) - .filter(suite -> ALLOWED_CIPHER_SUITES.contains(suite) && (acceptedCiphers.isEmpty() || acceptedCiphers.contains(suite))) + .filter(suite -> ALLOWED_CIPHER_SUITES.contains(suite) && acceptedCiphers.contains(suite)) .toArray(String[]::new); if (validCipherSuites.length == 0) { throw new IllegalStateException( @@ -117,7 +111,18 @@ public class DefaultTlsContext implements TlsContext { SSLParameters newParameters = sslContext.getDefaultSSLParameters(); newParameters.setCipherSuites(validCiphers); newParameters.setProtocols(validProtocols); - newParameters.setNeedClientAuth(true); + switch (peerAuthentication) { + case WANT: + newParameters.setWantClientAuth(true); + break; + case NEED: + newParameters.setNeedClientAuth(true); + break; + case DISABLED: + break; + default: + throw new UnsupportedOperationException("Unknown peer authentication: " + peerAuthentication); + } return newParameters; } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/MutableX509KeyManager.java b/security-utils/src/main/java/com/yahoo/security/tls/MutableX509KeyManager.java index a63ca28c793..02a32f79971 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/MutableX509KeyManager.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/MutableX509KeyManager.java @@ -20,25 +20,40 @@ import java.util.WeakHashMap; */ public class MutableX509KeyManager extends X509ExtendedKeyManager { - // Not using ThreadLocal as we want the x509 key manager instances to be collected + private final Object monitor = new Object(); + // Not using ThreadLocal as we want the thread local x509 key manager instances to be garbage collected // when either the thread dies or the MutableX509KeyManager instance is collected (latter not the case for ThreadLocal). private final WeakHashMap<Thread, X509ExtendedKeyManager> threadLocalManager = new WeakHashMap<>(); - private volatile X509ExtendedKeyManager currentManager; + private X509ExtendedKeyManager currentManager; public MutableX509KeyManager(KeyStore keystore, char[] password) { - this.currentManager = KeyManagerUtils.createDefaultX509KeyManager(keystore, password); + synchronized (monitor) { + this.currentManager = KeyManagerUtils.createDefaultX509KeyManager(keystore, password); + } } public MutableX509KeyManager() { - this.currentManager = KeyManagerUtils.createDefaultX509KeyManager(); + synchronized (monitor) { + this.currentManager = KeyManagerUtils.createDefaultX509KeyManager(); + } } public void updateKeystore(KeyStore keystore, char[] password) { - this.currentManager = KeyManagerUtils.createDefaultX509KeyManager(keystore, password); + synchronized (monitor) { + this.currentManager = KeyManagerUtils.createDefaultX509KeyManager(keystore, password); + } } public void useDefaultKeystore() { - this.currentManager = KeyManagerUtils.createDefaultX509KeyManager(); + synchronized (monitor) { + this.currentManager = KeyManagerUtils.createDefaultX509KeyManager(); + } + } + + public X509ExtendedKeyManager currentManager() { + synchronized (monitor) { + return currentManager; + } } @Override @@ -78,9 +93,11 @@ public class MutableX509KeyManager extends X509ExtendedKeyManager { } private X509ExtendedKeyManager updateAndGetThreadLocalManager() { - X509ExtendedKeyManager currentManager = this.currentManager; - threadLocalManager.put(Thread.currentThread(), currentManager); - return currentManager; + synchronized (monitor) { + X509ExtendedKeyManager currentManager = this.currentManager; + threadLocalManager.put(Thread.currentThread(), currentManager); + return currentManager; + } } @Override @@ -98,11 +115,12 @@ public class MutableX509KeyManager extends X509ExtendedKeyManager { } private X509ExtendedKeyManager getThreadLocalManager() { - X509ExtendedKeyManager manager = threadLocalManager.get(Thread.currentThread()); - if (manager == null) { - throw new IllegalStateException("Methods to retrieve valid aliases has not been called previously from this thread"); + synchronized (monitor) { + X509ExtendedKeyManager manager = threadLocalManager.get(Thread.currentThread()); + if (manager == null) { + throw new IllegalStateException("Methods to retrieve valid aliases has not been called previously from this thread"); + } + return manager; } - return manager; } - } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthentication.java b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthentication.java new file mode 100644 index 00000000000..9aa7b642b4a --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthentication.java @@ -0,0 +1,9 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security.tls; + +/** + * @author bjorncs + */ +public enum PeerAuthentication { + WANT, NEED, DISABLED +} diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java index b315dd00b31..ea26be0ef4f 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java @@ -4,6 +4,7 @@ package com.yahoo.security.tls; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLParameters; +import java.util.Set; /** * A simplified version of {@link SSLContext} modelled as an interface. @@ -12,6 +13,19 @@ import javax.net.ssl.SSLParameters; */ public interface TlsContext extends AutoCloseable { + Set<String> ALLOWED_CIPHER_SUITES = Set.of( + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_AES_128_GCM_SHA256", // TLSv1.3 + "TLS_AES_256_GCM_SHA384", // TLSv1.3 + "TLS_CHACHA20_POLY1305_SHA256"); // TLSv1.3 + + Set<String> ALLOWED_PROTOCOLS = Set.of("TLSv1.2"); // TODO Enable TLSv1.3 + SSLContext context(); SSLParameters parameters(); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java index a4e508e0d2a..f28cad2a071 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java @@ -66,7 +66,7 @@ public class TransportSecurityUtils { public static Optional<TlsContext> createTlsContext() { return getConfigFile() - .map(configFile -> new ReloadingTlsContext(configFile, getInsecureAuthorizationMode())); + .map(configFile -> new ConfigFileBasedTlsContext(configFile, getInsecureAuthorizationMode())); } private static Optional<String> getEnvironmentVariable(Map<String, String> environmentVariables, String variableName) { diff --git a/security-utils/src/test/java/com/yahoo/security/tls/ReloadingTlsContextTest.java b/security-utils/src/test/java/com/yahoo/security/tls/ConfigFileBasedTlsContextTest.java index f991f86fdce..4e6f0a141b0 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/ReloadingTlsContextTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/ConfigFileBasedTlsContextTest.java @@ -26,7 +26,7 @@ import static org.assertj.core.api.Assertions.assertThat; /** * @author bjorncs */ -public class ReloadingTlsContextTest { +public class ConfigFileBasedTlsContextTest { @Rule public TemporaryFolder tempDirectory = new TemporaryFolder(); @@ -55,12 +55,12 @@ public class ReloadingTlsContextTest { Path optionsFile = tempDirectory.newFile().toPath(); options.toJsonFile(optionsFile); - try (TlsContext tlsContext = new ReloadingTlsContext(optionsFile, AuthorizationMode.ENFORCE)) { + try (TlsContext tlsContext = new ConfigFileBasedTlsContext(optionsFile, AuthorizationMode.ENFORCE)) { SSLEngine sslEngine = tlsContext.createSslEngine(); assertThat(sslEngine).isNotNull(); String[] enabledCiphers = sslEngine.getEnabledCipherSuites(); assertThat(enabledCiphers).isNotEmpty(); - assertThat(enabledCiphers).isSubsetOf(DefaultTlsContext.ALLOWED_CIPHER_SUITES.toArray(new String[0])); + assertThat(enabledCiphers).isSubsetOf(TlsContext.ALLOWED_CIPHER_SUITES.toArray(new String[0])); String[] enabledProtocols = sslEngine.getEnabledProtocols(); assertThat(enabledProtocols).contains("TLSv1.2"); diff --git a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java index 5969d4d2ace..727a64ae934 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java @@ -15,7 +15,6 @@ import javax.security.auth.x500.X500Principal; import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Instant; -import java.util.List; import static com.yahoo.security.KeyAlgorithm.EC; import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; @@ -47,13 +46,13 @@ public class DefaultTlsContextTest { singletonList(new RequiredPeerCredential(RequiredPeerCredential.Field.CN, new HostGlobPattern("dummy")))))); DefaultTlsContext tlsContext = - new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, List.of()); + new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED); SSLEngine sslEngine = tlsContext.createSslEngine(); assertThat(sslEngine).isNotNull(); String[] enabledCiphers = sslEngine.getEnabledCipherSuites(); assertThat(enabledCiphers).isNotEmpty(); - assertThat(enabledCiphers).isSubsetOf(DefaultTlsContext.ALLOWED_CIPHER_SUITES.toArray(new String[0])); + assertThat(enabledCiphers).isSubsetOf(TlsContext.ALLOWED_CIPHER_SUITES.toArray(new String[0])); String[] enabledProtocols = sslEngine.getEnabledProtocols(); assertThat(enabledProtocols).contains("TLSv1.2"); diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.cpp b/storage/src/vespa/storage/distributor/idealstatemanager.cpp index eea6db8c782..6dbe6b1c2a5 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemanager.cpp @@ -9,6 +9,7 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storage/common/bucketmessages.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/vespalib/util/assert.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include "distributor_bucket_space_repo.h" #include "distributor_bucket_space.h" @@ -151,12 +152,13 @@ void IdealStateManager::verify_only_live_nodes_in_context(const StateChecker::Co const auto& state = c.systemState.getNodeState(lib::Node(lib::NodeType::STORAGE, index)); // Only nodes in Up, Initializing or Retired should ever be present in the DB. if (!state.getState().oneOf("uir")) { - LOG(warning, "%s in bucket DB is on node %u, which is in unavailable state %s. " - "Current cluster state is '%s'", - c.entry.getBucketId().toString().c_str(), - index, - state.getState().toString().c_str(), - c.systemState.toString().c_str()); + LOG(error, "%s in bucket DB is on node %u, which is in unavailable state %s. " + "Current cluster state is '%s'", + c.entry.getBucketId().toString().c_str(), + index, + state.getState().toString().c_str(), + c.systemState.toString().c_str()); + ASSERT_ONCE_OR_LOG(false, "Bucket DB contains replicas on unavailable node", 10000); _has_logged_phantom_replica_warning = true; } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index 2b0e50ed982..cab28e55b21 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -3,23 +3,17 @@ package com.yahoo.vespa.athenz.identity; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.KeyStoreType; import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.tls.AutoReloadingX509KeyManager; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.utils.SiaUtils; import javax.net.ssl.SSLContext; import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; -import java.time.Duration; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.logging.Logger; /** * A {@link ServiceIdentityProvider} that provides the credentials stored on file system. @@ -29,24 +23,19 @@ import java.util.logging.Logger; */ public class SiaIdentityProvider extends AbstractComponent implements ServiceIdentityProvider { - private static final Logger log = Logger.getLogger(SiaIdentityProvider.class.getName()); - - private static final Duration REFRESH_INTERVAL = Duration.ofHours(1); - - private final AtomicReference<SSLContext> sslContext = new AtomicReference<>(); + private final AutoReloadingX509KeyManager keyManager; + private final SSLContext sslContext; private final AthenzIdentity service; private final File privateKeyFile; private final File certificateFile; private final File trustStoreFile; - private final ScheduledExecutorService scheduler; @Inject public SiaIdentityProvider(SiaProviderConfig config) { this(new AthenzService(config.athenzDomain(), config.athenzService()), SiaUtils.getPrivateKeyFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())).toFile(), SiaUtils.getCertificateFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())).toFile(), - new File(config.trustStorePath()), - createScheduler()); + new File(config.trustStorePath())); } public SiaIdentityProvider(AthenzIdentity service, @@ -55,30 +44,19 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde this(service, SiaUtils.getPrivateKeyFile(siaPath, service).toFile(), SiaUtils.getCertificateFile(siaPath, service).toFile(), - trustStoreFile, - createScheduler()); + trustStoreFile); } public SiaIdentityProvider(AthenzIdentity service, File privateKeyFile, File certificateFile, - File trustStoreFile, - ScheduledExecutorService scheduler) { + File trustStoreFile) { this.service = service; this.privateKeyFile = privateKeyFile; this.certificateFile = certificateFile; this.trustStoreFile = trustStoreFile; - this.scheduler = scheduler; - this.sslContext.set(createIdentitySslContext()); - scheduler.scheduleAtFixedRate(this::reloadSslContext, REFRESH_INTERVAL.toMinutes(), REFRESH_INTERVAL.toMinutes(), TimeUnit.MINUTES); - } - - private static ScheduledThreadPoolExecutor createScheduler() { - return new ScheduledThreadPoolExecutor(1, runnable -> { - Thread thread = new Thread(runnable); - thread.setName("sia-identity-provider-sslcontext-updater"); - return thread; - }); + this.keyManager = AutoReloadingX509KeyManager.fromPemFiles(privateKeyFile.toPath(), certificateFile.toPath()); + this.sslContext = createIdentitySslContext(keyManager, trustStoreFile.toPath()); } @Override @@ -88,34 +66,18 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde @Override public SSLContext getIdentitySslContext() { - return sslContext.get(); + return sslContext; } - private SSLContext createIdentitySslContext() { + private static SSLContext createIdentitySslContext(AutoReloadingX509KeyManager keyManager, Path trustStoreFile) { return new SslContextBuilder() - .withTrustStore(trustStoreFile.toPath(), KeyStoreType.JKS) - .withKeyStore(privateKeyFile.toPath(), certificateFile.toPath()) + .withTrustStore(trustStoreFile, KeyStoreType.JKS) + .withKeyManager(keyManager) .build(); } - private void reloadSslContext() { - log.log(LogLevel.DEBUG, "Updating SSLContext for identity " + service.getFullName()); - try { - SSLContext sslContext = createIdentitySslContext(); - this.sslContext.set(sslContext); - } catch (Exception e) { - log.log(LogLevel.SEVERE, "Failed to update SSLContext: " + e.getMessage(), e); - } - } - - @Override public void deconstruct() { - try { - scheduler.shutdownNow(); - scheduler.awaitTermination(90, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + keyManager.close(); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java index a1d8a9ca258..d4494c1bd26 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import javax.net.ssl.SSLContext; import java.security.KeyPair; import java.security.cert.X509Certificate; @@ -15,16 +14,13 @@ class AthenzCredentials { private final X509Certificate certificate; private final KeyPair keyPair; private final SignedIdentityDocument identityDocument; - private final SSLContext identitySslContext; AthenzCredentials(X509Certificate certificate, KeyPair keyPair, - SignedIdentityDocument identityDocument, - SSLContext identitySslContext) { + SignedIdentityDocument identityDocument) { this.certificate = certificate; this.keyPair = keyPair; this.identityDocument = identityDocument; - this.identitySslContext = identitySslContext; } X509Certificate getCertificate() { @@ -39,7 +35,4 @@ class AthenzCredentials { return identityDocument; } - SSLContext getIdentitySslContext() { - return identitySslContext; - } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java index 39d0db4affd..9e2d8bc548c 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; -import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; @@ -14,12 +14,10 @@ import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; -import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.utils.SiaUtils; import com.yahoo.vespa.defaults.Defaults; import javax.net.ssl.SSLContext; -import java.io.File; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -31,7 +29,6 @@ import java.time.Clock; import java.time.Duration; import java.util.Optional; -import static com.yahoo.security.KeyStoreType.JKS; import static java.util.Collections.singleton; /** @@ -49,14 +46,12 @@ class AthenzCredentialsService { private final URI ztsEndpoint; private final AthenzService configserverIdentity; private final ServiceIdentityProvider nodeIdentityProvider; - private final File trustStoreJks; private final String hostname; private final CsrGenerator csrGenerator; private final Clock clock; AthenzCredentialsService(IdentityConfig identityConfig, ServiceIdentityProvider nodeIdentityProvider, - File trustStoreJks, String hostname, Clock clock) { this.tenantIdentity = new AthenzService(identityConfig.domain(), identityConfig.service()); @@ -64,7 +59,6 @@ class AthenzCredentialsService { this.ztsEndpoint = URI.create(identityConfig.ztsUrl()); this.configserverIdentity = new AthenzService(identityConfig.configserverIdentityName()); this.nodeIdentityProvider = nodeIdentityProvider; - this.trustStoreJks = trustStoreJks; this.hostname = hostname; this.csrGenerator = new CsrGenerator(identityConfig.athenzDnsSuffix(), identityConfig.configserverIdentityName()); this.clock = clock; @@ -94,9 +88,8 @@ class AthenzCredentialsService { false, csr); X509Certificate certificate = instanceIdentity.certificate(); - SSLContext identitySslContext = createIdentitySslContext(keyPair.getPrivate(), certificate); writeCredentialsToDisk(keyPair.getPrivate(), certificate, document); - return new AthenzCredentials(certificate, keyPair, document, identitySslContext); + return new AthenzCredentials(certificate, keyPair, document); } } @@ -117,9 +110,8 @@ class AthenzCredentialsService { false, csr); X509Certificate certificate = instanceIdentity.certificate(); - SSLContext identitySslContext = createIdentitySslContext(newKeyPair.getPrivate(), certificate); writeCredentialsToDisk(newKeyPair.getPrivate(), certificate, document); - return new AthenzCredentials(certificate, newKeyPair, document, identitySslContext); + return new AthenzCredentials(certificate, newKeyPair, document); } } @@ -134,8 +126,7 @@ class AthenzCredentialsService { if (Files.notExists(IDENTITY_DOCUMENT_FILE)) return Optional.empty(); SignedIdentityDocument signedIdentityDocument = EntityBindingsMapper.readSignedIdentityDocumentFromFile(IDENTITY_DOCUMENT_FILE); KeyPair keyPair = new KeyPair(KeyUtils.extractPublicKey(privateKey.get()), privateKey.get()); - SSLContext sslContext = createIdentitySslContext(privateKey.get(), certificate.get()); - return Optional.of(new AthenzCredentials(certificate.get(), keyPair, signedIdentityDocument, sslContext)); + return Optional.of(new AthenzCredentials(certificate.get(), keyPair, signedIdentityDocument)); } private boolean isExpired(X509Certificate certificate) { @@ -150,13 +141,6 @@ class AthenzCredentialsService { EntityBindingsMapper.writeSignedIdentityDocumentToFile(IDENTITY_DOCUMENT_FILE, identityDocument); } - private SSLContext createIdentitySslContext(PrivateKey privateKey, X509Certificate certificate) { - return new SslContextBuilder() - .withKeyStore(privateKey, certificate) - .withTrustStore(trustStoreJks.toPath(), JKS) - .build(); - } - private DefaultIdentityDocumentClient createIdentityDocumentClient() { return new DefaultIdentityDocumentClient( configserverEndpoint, diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index ac255289883..bff40d67fa6 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -12,8 +12,11 @@ import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; +import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; +import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.tls.MutableX509KeyManager; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzService; @@ -22,13 +25,14 @@ import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identity.SiaIdentityProvider; -import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.utils.SiaUtils; import com.yahoo.vespa.defaults.Defaults; import javax.net.ssl.SSLContext; -import java.io.File; +import javax.net.ssl.X509ExtendedKeyManager; import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.time.Clock; @@ -42,6 +46,9 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.logging.Logger; +import static com.yahoo.security.KeyStoreType.JKS; +import static com.yahoo.security.KeyStoreType.PKCS12; + /** * A {@link AthenzIdentityProvider} / {@link ServiceIdentityProvider} component that provides the tenant identity. * @@ -59,10 +66,14 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private final static Duration ROLE_SSL_CONTEXT_EXPIRY = Duration.ofHours(24); private final static Duration ROLE_TOKEN_EXPIRY = Duration.ofMinutes(30); + // TODO Make path to trust store config + private static final Path DEFAULT_TRUST_STORE = Paths.get(Defaults.getDefaults().underVespaHome("share/ssl/certs/yahoo_certificate_bundle.jks")); + public static final String CERTIFICATE_EXPIRY_METRIC_NAME = "athenz-tenant-cert.expiry.seconds"; private volatile AthenzCredentials credentials; private final Metric metric; + private final Path trustStore; private final AthenzCredentialsService athenzCredentialsService; private final ScheduledExecutorService scheduler; private final Clock clock; @@ -70,6 +81,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private final String dnsSuffix; private final URI ztsEndpoint; + private final MutableX509KeyManager identityKeyManager = new MutableX509KeyManager(); + private final SSLContext identitySslContext; private final LoadingCache<AthenzRole, SSLContext> roleSslContextCache; private final LoadingCache<AthenzRole, ZToken> roleSpecificRoleTokenCache; private final LoadingCache<AthenzDomain, ZToken> domainSpecificRoleTokenCache; @@ -79,9 +92,9 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen public AthenzIdentityProviderImpl(IdentityConfig config, Metric metric) { this(config, metric, + DEFAULT_TRUST_STORE, new AthenzCredentialsService(config, - createNodeIdentityProvider(config), - getDefaultTrustStoreLocation(), + createNodeIdentityProvider(config, DEFAULT_TRUST_STORE), Defaults.getDefaults().vespaHostname(), Clock.systemUTC()), new ScheduledThreadPoolExecutor(1), @@ -92,10 +105,12 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen AthenzIdentityProviderImpl(IdentityConfig config, Metric metric, + Path trustStore, AthenzCredentialsService athenzCredentialsService, ScheduledExecutorService scheduler, Clock clock) { this.metric = metric; + this.trustStore = trustStore; this.athenzCredentialsService = athenzCredentialsService; this.scheduler = scheduler; this.clock = clock; @@ -106,6 +121,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen roleSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); domainSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); this.csrGenerator = new CsrGenerator(config.athenzDnsSuffix(), config.configserverIdentityName()); + this.identitySslContext = createIdentitySslContext(identityKeyManager, trustStore); registerInstance(); } @@ -121,11 +137,18 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen }); } + private static SSLContext createIdentitySslContext(X509ExtendedKeyManager keyManager, Path trustStore) { + return new SslContextBuilder() + .withKeyManager(keyManager) + .withTrustStore(trustStore, JKS) + .build(); + } + private void registerInstance() { try { - credentials = athenzCredentialsService.registerInstance(); - scheduler.scheduleAtFixedRate(this::refreshCertificate, UPDATE_PERIOD.toMinutes(), UPDATE_PERIOD.toMinutes(), TimeUnit.MINUTES); - scheduler.scheduleAtFixedRate(this::reportMetrics, 0, 5, TimeUnit.MINUTES); + updateIdentityCredentials(this.athenzCredentialsService.registerInstance()); + this.scheduler.scheduleAtFixedRate(this::refreshCertificate, UPDATE_PERIOD.toMinutes(), UPDATE_PERIOD.toMinutes(), TimeUnit.MINUTES); + this.scheduler.scheduleAtFixedRate(this::reportMetrics, 0, 5, TimeUnit.MINUTES); } catch (Throwable t) { throw new AthenzIdentityProviderException("Could not retrieve Athenz credentials", t); } @@ -148,7 +171,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen @Override public SSLContext getIdentitySslContext() { - return credentials.getIdentitySslContext(); + return identitySslContext; } @Override @@ -189,13 +212,22 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen return Collections.singletonList(credentials.getCertificate()); } + private void updateIdentityCredentials(AthenzCredentials credentials) { + this.credentials = credentials; + this.identityKeyManager.updateKeystore( + KeyStoreBuilder.withType(PKCS12) + .withKeyEntry("default", credentials.getKeyPair().getPrivate(), credentials.getCertificate()) + .build(), + new char[0]); + } + private SSLContext createRoleSslContext(AthenzRole role) { Pkcs10Csr csr = csrGenerator.generateRoleCsr(identity, role, credentials.getIdentityDocument().providerUniqueId(), credentials.getKeyPair()); try (ZtsClient client = createZtsClient()) { X509Certificate roleCertificate = client.getRoleCertificate(role, csr); return new SslContextBuilder() .withKeyStore(credentials.getKeyPair().getPrivate(), roleCertificate) - .withTrustStore(getDefaultTrustStoreLocation().toPath(), KeyStoreType.JKS) + .withTrustStore(trustStore, KeyStoreType.JKS) .build(); } } @@ -226,13 +258,9 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } } - private static SiaIdentityProvider createNodeIdentityProvider(IdentityConfig config) { + private static SiaIdentityProvider createNodeIdentityProvider(IdentityConfig config, Path trustStore) { return new SiaIdentityProvider( - new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, getDefaultTrustStoreLocation()); - } - - private static File getDefaultTrustStoreLocation() { - return new File(Defaults.getDefaults().underVespaHome("share/ssl/certs/yahoo_certificate_bundle.jks")); + new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, trustStore.toFile()); } private boolean isExpired(AthenzCredentials credentials) { @@ -245,9 +273,9 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen void refreshCertificate() { try { - credentials = isExpired(credentials) - ? athenzCredentialsService.registerInstance() - : athenzCredentialsService.updateCredentials(credentials.getIdentityDocument(), credentials.getIdentitySslContext()); + updateIdentityCredentials(isExpired(credentials) + ? athenzCredentialsService.registerInstance() + : athenzCredentialsService.updateCredentials(credentials.getIdentityDocument(), identitySslContext)); } catch (Throwable t) { log.log(LogLevel.WARNING, "Failed to update credentials: " + t.getMessage(), t); } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java index 0195d6000e1..31152a4602f 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java @@ -24,10 +24,8 @@ import java.security.KeyStore; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.util.concurrent.ScheduledExecutorService; import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.mock; /** * @author bjorncs @@ -55,8 +53,7 @@ public class SiaIdentityProviderTest { new AthenzService("domain", "service-name"), keyFile, certificateFile, - trustStoreFile, - mock(ScheduledExecutorService.class)); + trustStoreFile); assertNotNull(provider.getIdentitySslContext()); } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java index 01dab2dada3..c584b803815 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java @@ -4,14 +4,30 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyStoreBuilder; +import com.yahoo.security.KeyStoreType; +import com.yahoo.security.KeyStoreUtils; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.Pkcs10Csr; +import com.yahoo.security.Pkcs10CsrBuilder; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; import com.yahoo.test.ManualClock; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import javax.security.auth.x500.X500Principal; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Path; +import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.concurrent.ScheduledExecutorService; import java.util.function.Supplier; @@ -43,13 +59,36 @@ public class AthenzIdentityProviderImplTest { .ztsUrl("https:localhost:4443/zts/v1") .athenzDnsSuffix("dev-us-north-1.vespa.cloud")); + private final KeyPair caKeypair = KeyUtils.generateKeypair(KeyAlgorithm.EC); + private Path trustStoreFile; + private X509Certificate caCertificate; + + @Before + public void createTrustStoreFile() throws IOException { + caCertificate = X509CertificateBuilder + .fromKeypair( + caKeypair, + new X500Principal("CN=mydummyca"), + Instant.EPOCH, + Instant.EPOCH.plus(10000, ChronoUnit.DAYS), + SignatureAlgorithm.SHA256_WITH_ECDSA, + BigInteger.ONE) + .build(); + trustStoreFile = tempDir.newFile().toPath(); + KeyStoreUtils.writeKeyStoreToFile( + KeyStoreBuilder.withType(KeyStoreType.JKS) + .withKeyEntry("default", caKeypair.getPrivate(), caCertificate) + .build(), + trustStoreFile); + } + @Test(expected = AthenzIdentityProviderException.class) public void component_creation_fails_when_credentials_not_found() { AthenzCredentialsService credentialService = mock(AthenzCredentialsService.class); when(credentialService.registerInstance()) .thenThrow(new RuntimeException("athenz unavailable")); - new AthenzIdentityProviderImpl(IDENTITY_CONFIG, mock(Metric.class), credentialService, mock(ScheduledExecutorService.class), new ManualClock(Instant.EPOCH)); + new AthenzIdentityProviderImpl(IDENTITY_CONFIG, mock(Metric.class), trustStoreFile ,credentialService, mock(ScheduledExecutorService.class), new ManualClock(Instant.EPOCH)); } @Test @@ -59,18 +98,19 @@ public class AthenzIdentityProviderImplTest { AthenzCredentialsService athenzCredentialsService = mock(AthenzCredentialsService.class); - X509Certificate certificate = getCertificate(getExpirationSupplier(clock)); + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC); + X509Certificate certificate = getCertificate(keyPair, getExpirationSupplier(clock)); when(athenzCredentialsService.registerInstance()) - .thenReturn(new AthenzCredentials(certificate, null, null, null)); + .thenReturn(new AthenzCredentials(certificate, keyPair, null)); when(athenzCredentialsService.updateCredentials(any(), any())) .thenThrow(new RuntimeException("#1")) .thenThrow(new RuntimeException("#2")) - .thenReturn(new AthenzCredentials(certificate, null, null, null)); + .thenReturn(new AthenzCredentials(certificate, keyPair, null)); AthenzIdentityProviderImpl identityProvider = - new AthenzIdentityProviderImpl(IDENTITY_CONFIG, metric, athenzCredentialsService, mock(ScheduledExecutorService.class), clock); + new AthenzIdentityProviderImpl(IDENTITY_CONFIG, metric, trustStoreFile, athenzCredentialsService, mock(ScheduledExecutorService.class), clock); identityProvider.reportMetrics(); verify(metric).set(eq(AthenzIdentityProviderImpl.CERTIFICATE_EXPIRY_METRIC_NAME), eq(certificateValidity.getSeconds()), any()); @@ -99,10 +139,18 @@ public class AthenzIdentityProviderImplTest { return () -> new Date(clock.instant().plus(certificateValidity).toEpochMilli()); } - private X509Certificate getCertificate(Supplier<Date> expiry) { - X509Certificate x509Certificate = mock(X509Certificate.class); - when(x509Certificate.getNotAfter()).thenReturn(expiry.get()); - return x509Certificate; + private X509Certificate getCertificate(KeyPair keyPair, Supplier<Date> expiry) { + Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=dummy"), keyPair, SignatureAlgorithm.SHA256_WITH_ECDSA) + .build(); + return X509CertificateBuilder + .fromCsr(csr, + caCertificate.getSubjectX500Principal(), + Instant.EPOCH, + expiry.get().toInstant(), + caKeypair.getPrivate(), + SignatureAlgorithm.SHA256_WITH_ECDSA, + BigInteger.ONE) + .build(); } } diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 999bb7bcc01..1ee29c36b7d 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -126,16 +126,46 @@ </goals> <configuration> <minimizeJar>false</minimizeJar> + <relocations> <relocation> - <pattern>org.apache.http</pattern> - <shadedPattern>com.yahoo.vespa.feeder.shaded.internal.apache.http</shadedPattern> + <pattern>com.google</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>commons-codec</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>commons-logging</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + <excludes> + <exclude>org.apache.hadoop.**</exclude> + <exclude>org.apache.pig.**</exclude> + </excludes> </relocation> <relocation> - <pattern>org.apache.commons</pattern> - <shadedPattern>com.yahoo.vespa.feeder.shaded.internal.apache.commons</shadedPattern> + <pattern>com.fasterxml</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>org.codehaus</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>io.airlift</pattern> + <shadedPattern>shaded.vespa</shadedPattern> + </relocation> + <relocation> + <pattern>com.ctc.wstx</pattern> + <shadedPattern>shaded.vespa</shadedPattern> </relocation> </relocations> + </configuration> </execution> </executions> diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java index 008f3b63a89..fff0aa910d5 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java @@ -13,13 +13,9 @@ import java.util.concurrent.TimeUnit; */ public final class FeedParams { - public boolean getDenyIfBusyV3() { - return denyIfBusyV3; - } + public boolean getDenyIfBusyV3() { return denyIfBusyV3; } - public long getMaxSleepTimeMs() { - return maxSleepTimeMs; - } + public long getMaxSleepTimeMs() { return maxSleepTimeMs; } public boolean getSilentUpgrade() { return silentUpgrade; } @@ -36,6 +32,7 @@ public final class FeedParams { * Mutable class used to instantiate a {@link FeedParams}. */ public static final class Builder { + private DataFormat dataFormat = DataFormat.JSON_UTF8; private long serverTimeout = TimeUnit.SECONDS.toMillis(180); private long clientTimeout = TimeUnit.SECONDS.toMillis(20); @@ -57,7 +54,7 @@ public final class FeedParams { * @return this, for chaining */ @Beta - public Builder withSilentUpgrade(boolean silentUpgrade) { + public Builder setSilentUpgrade(boolean silentUpgrade) { this.silentUpgrade = silentUpgrade; return this; } @@ -165,6 +162,7 @@ public final class FeedParams { /** * Sets the maximum number of operations to be in-flight. + * * @param maxInFlightRequests max number of operations. * @return this, for chaining */ @@ -246,11 +244,14 @@ public final class FeedParams { return maxChunkSizeBytes; } - public int getmaxInFlightRequests() { + public int getMaxInFlightRequests() { return maxInFlightRequests; } + } + // NOTE! See toBuilder at the end of this class if you add fields here + private final DataFormat dataFormat; private final long serverTimeoutMillis; private final long clientTimeoutMillis; @@ -263,7 +264,6 @@ public final class FeedParams { private final long maxSleepTimeMs; private final boolean silentUpgrade; - private FeedParams(DataFormat dataFormat, long serverTimeout, long clientTimeout, String route, int maxChunkSizeBytes, final int maxInFlightRequests, long localQueueTimeOut, String priority, boolean denyIfBusyV3, long maxSleepTimeMs, @@ -319,4 +319,20 @@ public final class FeedParams { return localQueueTimeOut; } + /** Returns a builder initialized to the values of this */ + public FeedParams.Builder toBuilder() { + Builder b = new Builder(); + b.setDataFormat(dataFormat); + b.setServerTimeout(serverTimeoutMillis, TimeUnit.MILLISECONDS); + b.setClientTimeout(clientTimeoutMillis, TimeUnit.MILLISECONDS); + b.setRoute(route); + b.setMaxChunkSizeBytes(maxChunkSizeBytes); + b.setMaxInFlightRequests(maxInFlightRequests); + b.setPriority(priority); + b.setDenyIfBusyV3(denyIfBusyV3); + b.setMaxSleepTimeMs(maxSleepTimeMs); + b.setSilentUpgrade(silentUpgrade); + return b; + } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java index 48fd21e2b1f..4e1406ab966 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java @@ -133,6 +133,8 @@ public final class SessionParams { } } + // NOTE! See toBuilder at the end of this class if you add fields here + private final List<Cluster> clusters; private final FeedParams feedParams; private final ConnectionParams connectionParams; @@ -179,4 +181,15 @@ public final class SessionParams { return errorReport; } + public Builder toBuilder() { + Builder b = new Builder(); + clusters.forEach(c -> b.addCluster(c)); + b.setFeedParams(feedParams); + b.setConnectionParams(connectionParams); + b.setClientQueueSize(clientQueueSize); + b.setErrorReporter(errorReport); + b.setThrottlerMinSize(throttlerMinSize); + return b; + } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java index da45acc5687..6e1f3419e8e 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.http.client.config.Cluster; import com.yahoo.vespa.http.client.config.ConnectionParams; import com.yahoo.vespa.http.client.config.Endpoint; import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; import com.yahoo.vespa.http.client.core.Document; import com.yahoo.vespa.http.client.core.Exceptions; import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; @@ -25,45 +24,35 @@ import java.util.concurrent.TimeUnit; */ public class ClusterConnection implements AutoCloseable { - private final OperationProcessor operationProcessor; private final List<IOThread> ioThreads = new ArrayList<>(); private final int clusterId; - private final SessionParams.ErrorReporter errorReporter; private static JsonFactory jsonFactory = new JsonFactory(); private static ObjectMapper objectMapper = new ObjectMapper(); - public ClusterConnection( - OperationProcessor operationProcessor, - FeedParams feedParams, - ConnectionParams connectionParams, - SessionParams.ErrorReporter errorReporter, - Cluster cluster, - int clusterId, - int clientQueueSizePerCluster, - ScheduledThreadPoolExecutor timeoutExecutor) { - this.errorReporter = errorReporter; - if (cluster.getEndpoints().isEmpty()) { + public ClusterConnection(OperationProcessor operationProcessor, + FeedParams feedParams, + ConnectionParams connectionParams, + Cluster cluster, + int clusterId, + int clientQueueSizePerCluster, + ScheduledThreadPoolExecutor timeoutExecutor) { + if (cluster.getEndpoints().isEmpty()) throw new IllegalArgumentException("Cannot feed to empty cluster."); - } - this.operationProcessor = operationProcessor; + this.clusterId = clusterId; - final int totalNumberOfEndpointsInThisCluster = cluster.getEndpoints().size() - * connectionParams.getNumPersistentConnectionsPerEndpoint(); - if (totalNumberOfEndpointsInThisCluster == 0) { - return; - } + int totalNumberOfEndpointsInThisCluster = cluster.getEndpoints().size() * connectionParams.getNumPersistentConnectionsPerEndpoint(); + if (totalNumberOfEndpointsInThisCluster == 0) return; + // Lower than 1 does not make any sense. - final int maxInFlightPerSession = Math.max( - 1, feedParams.getMaxInFlightRequests() / totalNumberOfEndpointsInThisCluster); + int maxInFlightPerSession = Math.max(1, feedParams.getMaxInFlightRequests() / totalNumberOfEndpointsInThisCluster); + DocumentQueue documentQueue = null; for (Endpoint endpoint : cluster.getEndpoints()) { - final EndpointResultQueue endpointResultQueue = new EndpointResultQueue( - operationProcessor, - endpoint, - clusterId, - timeoutExecutor, - feedParams.getServerTimeout(TimeUnit.MILLISECONDS) - + feedParams.getClientTimeout(TimeUnit.MILLISECONDS)); + EndpointResultQueue endpointResultQueue = new EndpointResultQueue(operationProcessor, + endpoint, + clusterId, + timeoutExecutor, + feedParams.getServerTimeout(TimeUnit.MILLISECONDS) + feedParams.getClientTimeout(TimeUnit.MILLISECONDS)); for (int i = 0; i < connectionParams.getNumPersistentConnectionsPerEndpoint(); i++) { GatewayConnection gatewayConnection; if (connectionParams.isDryRun()) { @@ -74,24 +63,22 @@ public class ClusterConnection implements AutoCloseable { feedParams, cluster.getRoute(), connectionParams, - new ApacheGatewayConnection.HttpClientFactory( - connectionParams, endpoint.isUseSsl()), + new ApacheGatewayConnection.HttpClientFactory(connectionParams, endpoint.isUseSsl()), operationProcessor.getClientId() ); } if (documentQueue == null) { documentQueue = new DocumentQueue(clientQueueSizePerCluster); } - final IOThread ioThread = new IOThread( - operationProcessor.getIoThreadGroup(), - endpointResultQueue, - gatewayConnection, - clusterId, - feedParams.getMaxChunkSizeBytes(), - maxInFlightPerSession, - feedParams.getLocalQueueTimeOut(), - documentQueue, - feedParams.getMaxSleepTimeMs()); + IOThread ioThread = new IOThread(operationProcessor.getIoThreadGroup(), + endpointResultQueue, + gatewayConnection, + clusterId, + feedParams.getMaxChunkSizeBytes(), + maxInFlightPerSession, + feedParams.getLocalQueueTimeOut(), + documentQueue, + feedParams.getMaxSleepTimeMs()); ioThreads.add(ioThread); } } @@ -103,9 +90,9 @@ public class ClusterConnection implements AutoCloseable { public void post(Document document) throws EndpointIOException { String documentIdStr = document.getDocumentId(); - //the same document ID must always go to the same destination + // The same document ID must always go to the same destination // In noHandshakeMode this has no effect as the documentQueue is shared between the IOThreads. - int hash = documentIdStr.hashCode() & 0x7FFFFFFF; //strip sign bit + int hash = documentIdStr.hashCode() & 0x7FFFFFFF; // Strip sign bit IOThread ioThread = ioThreads.get(hash % ioThreads.size()); try { ioThread.post(document); @@ -148,7 +135,7 @@ public class ClusterConnection implements AutoCloseable { } public String getStatsAsJSon() throws IOException { - final StringWriter stringWriter = new StringWriter(); + StringWriter stringWriter = new StringWriter(); JsonGenerator jsonGenerator = jsonFactory.createGenerator(stringWriter); jsonGenerator.writeStartObject(); jsonGenerator.writeArrayFieldStart("session"); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java index 8c4ff3ae108..8ec4f6cb7f4 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java @@ -24,7 +24,7 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * Class for handling asynchronous feeding of new documents and processing of results. + * Thread which feeds document operations asynchronously and processes the results. * * @author Einar M R Rosenvinge */ @@ -53,19 +53,18 @@ class IOThread implements Runnable, AutoCloseable { private final AtomicInteger docsReceivedCounter = new AtomicInteger(0); private final AtomicInteger statusReceivedCounter = new AtomicInteger(0); private final AtomicInteger pendingDocumentStatusCount = new AtomicInteger(0); - private final AtomicInteger successfullHandshakes = new AtomicInteger(0); + private final AtomicInteger successfulHandshakes = new AtomicInteger(0); private final AtomicInteger lastGatewayProcessTimeMillis = new AtomicInteger(0); - IOThread( - ThreadGroup ioThreadGroup, - EndpointResultQueue endpointResultQueue, - GatewayConnection client, - int clusterId, - int maxChunkSizeBytes, - int maxInFlightRequests, - long localQueueTimeOut, - DocumentQueue documentQueue, - long maxSleepTimeMs) { + IOThread(ThreadGroup ioThreadGroup, + EndpointResultQueue endpointResultQueue, + GatewayConnection client, + int clusterId, + int maxChunkSizeBytes, + int maxInFlightRequests, + long localQueueTimeOut, + DocumentQueue documentQueue, + long maxSleepTimeMs) { this.documentQueue = documentQueue; this.endpoint = client.getEndpoint(); this.client = client; @@ -86,6 +85,9 @@ class IOThread implements Runnable, AutoCloseable { } public static class ConnectionStats { + + // NOTE: These fields are accessed by reflection in JSON serialization + public final int wrongSessionDetectedCounter; public final int wrongVersionDetectedCounter; public final int problemStatusCodeFromServerCounter; @@ -96,16 +98,15 @@ class IOThread implements Runnable, AutoCloseable { public final int successfullHandshakes; public final int lastGatewayProcessTimeMillis; - protected ConnectionStats( - final int wrongSessionDetectedCounter, - final int wrongVersionDetectedCounter, - final int problemStatusCodeFromServerCounter, - final int executeProblemsCounter, - final int docsReceivedCounter, - final int statusReceivedCounter, - final int pendingDocumentStatusCount, - final int successfullHandshakes, - final int lastGatewayProcessTimeMillis) { + ConnectionStats(int wrongSessionDetectedCounter, + int wrongVersionDetectedCounter, + int problemStatusCodeFromServerCounter, + int executeProblemsCounter, + int docsReceivedCounter, + int statusReceivedCounter, + int pendingDocumentStatusCount, + int successfullHandshakes, + int lastGatewayProcessTimeMillis) { this.wrongSessionDetectedCounter = wrongSessionDetectedCounter; this.wrongVersionDetectedCounter = wrongVersionDetectedCounter; this.problemStatusCodeFromServerCounter = problemStatusCodeFromServerCounter; @@ -130,16 +131,14 @@ class IOThread implements Runnable, AutoCloseable { docsReceivedCounter.get(), statusReceivedCounter.get(), pendingDocumentStatusCount.get(), - successfullHandshakes.get(), + successfulHandshakes.get(), lastGatewayProcessTimeMillis.get()); } @Override public void close() { documentQueue.close(); - if (stopSignal.getCount() == 0) { - return; - } + if (stopSignal.getCount() == 0) return; stopSignal.countDown(); log.finer("Closed called."); @@ -166,8 +165,7 @@ class IOThread implements Runnable, AutoCloseable { log.fine("Session to " + endpoint + " closed."); } - - public void post(final Document document) throws InterruptedException { + public void post(Document document) throws InterruptedException { documentQueue.put(document, Thread.currentThread().getThreadGroup() == ioThreadGroup); } @@ -177,8 +175,8 @@ class IOThread implements Runnable, AutoCloseable { } - List<Document> getNextDocsForFeeding(int maxWaitUnits, TimeUnit timeUnit) { - final List<Document> docsForSendChunk = new ArrayList<>(); + List<Document> getNextDocsForFeeding(long maxWaitUnits, TimeUnit timeUnit) { + List<Document> docsForSendChunk = new ArrayList<>(); int chunkSizeBytes = 0; try { drainFirstDocumentsInQueueIfOld(); @@ -214,8 +212,7 @@ class IOThread implements Runnable, AutoCloseable { } } - private void markDocumentAsFailed( - List<Document> docs, ServerResponseException servletException) { + private void markDocumentAsFailed(List<Document> docs, ServerResponseException servletException) { for (Document doc : docs) { resultQueue.failOperation( EndPointResultFactory.createTransientError( @@ -223,8 +220,7 @@ class IOThread implements Runnable, AutoCloseable { } } - private InputStream sendAndReceive(List<Document> docs) - throws IOException, ServerResponseException { + private InputStream sendAndReceive(List<Document> docs) throws IOException, ServerResponseException { try { // Post the new docs and get async responses for other posts. return client.writeOperations(docs); @@ -238,17 +234,19 @@ class IOThread implements Runnable, AutoCloseable { } private static class ProcessResponse { + private final int transitiveErrorCount; private final int processResultsCount; + ProcessResponse(int transitiveErrorCount, int processResultsCount) { this.transitiveErrorCount = transitiveErrorCount; this.processResultsCount = processResultsCount; } + } private ProcessResponse processResponse(InputStream serverResponse) throws IOException { - final Collection<EndpointResult> endpointResults = - EndPointResultFactory.createResult(endpoint, serverResponse); + Collection<EndpointResult> endpointResults = EndPointResultFactory.createResult(endpoint, serverResponse); statusReceivedCounter.addAndGet(endpointResults.size()); int transientErrors = 0; for (EndpointResult endpointResult : endpointResults) { @@ -271,15 +269,14 @@ class IOThread implements Runnable, AutoCloseable { return processResponse; } - private ProcessResponse pullAndProcessData(int maxWaitTimeMilliSecs) - throws ServerResponseException, IOException { - final int pendingResultQueueSize = resultQueue.getPendingSize(); + private ProcessResponse pullAndProcessData(long maxWaitTimeMs) throws ServerResponseException, IOException { + int pendingResultQueueSize = resultQueue.getPendingSize(); pendingDocumentStatusCount.set(pendingResultQueueSize); - List<Document> nextDocsForFeeding = (pendingResultQueueSize > maxInFlightRequests) + List<Document> nextDocsForFeeding = + (pendingResultQueueSize > maxInFlightRequests) ? new ArrayList<>() // The queue is full, will not send more documents. - : getNextDocsForFeeding(maxWaitTimeMilliSecs, TimeUnit.MILLISECONDS); - + : getNextDocsForFeeding(maxWaitTimeMs, TimeUnit.MILLISECONDS); if (nextDocsForFeeding.isEmpty() && pendingResultQueueSize == 0) { //we have no unfinished business with the server now. @@ -288,6 +285,7 @@ class IOThread implements Runnable, AutoCloseable { } log.finest("Awaiting " + pendingResultQueueSize + " results."); ProcessResponse processResponse = feedDocumentAndProcessResults(nextDocsForFeeding); + if (pendingResultQueueSize > maxInFlightRequests && processResponse.processResultsCount == 0) { try { // Max outstanding document operations, no more results on server side, wait a bit @@ -319,7 +317,7 @@ class IOThread implements Runnable, AutoCloseable { case CONNECTED: try { client.handshake(); - successfullHandshakes.getAndIncrement(); + successfulHandshakes.getAndIncrement(); } catch (ServerResponseException ser) { executeProblemsCounter.incrementAndGet(); log.info("Handshake did not work out " + endpoint + ": " + Exceptions.toMessageString(ser)); @@ -337,7 +335,7 @@ class IOThread implements Runnable, AutoCloseable { return ThreadState.SESSION_SYNCED; case SESSION_SYNCED: try { - ProcessResponse processResponse = pullAndProcessData(100); + ProcessResponse processResponse = pullAndProcessData(1); gatewayThrottler.handleCall(processResponse.transitiveErrorCount); } catch (ServerResponseException ser) { @@ -387,9 +385,8 @@ class IOThread implements Runnable, AutoCloseable { private void drainFirstDocumentsInQueueIfOld() { while (true) { Optional<Document> document = documentQueue.pollDocumentIfTimedoutInQueue(localQueueTimeOut); - if (! document.isPresent()) { - return; - } + if ( ! document.isPresent()) return; + EndpointResult endpointResult = EndPointResultFactory.createTransientError( endpoint, document.get().getOperationId(), new Exception("Not sending document operation, timed out in queue after " diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java index 45133901567..692d90abe50 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java @@ -56,41 +56,34 @@ public class OperationProcessor { private final ThreadGroup ioThreadGroup; private final String clientId = new BigInteger(130, random).toString(32); - public OperationProcessor( - IncompleteResultsThrottler incompleteResultsThrottler, - FeedClient.ResultCallback resultCallback, - SessionParams sessionParams, - ScheduledThreadPoolExecutor timeoutExecutor) { + public OperationProcessor(IncompleteResultsThrottler incompleteResultsThrottler, + FeedClient.ResultCallback resultCallback, + SessionParams sessionParams, + ScheduledThreadPoolExecutor timeoutExecutor) { this.numDestinations = sessionParams.getClusters().size(); this.resultCallback = resultCallback; this.incompleteResultsThrottler = incompleteResultsThrottler; this.timeoutExecutor = timeoutExecutor; this.ioThreadGroup = new ThreadGroup("operationprocessor"); - if (sessionParams.getClusters().isEmpty()) { + if (sessionParams.getClusters().isEmpty()) throw new IllegalArgumentException("Cannot feed to 0 clusters."); - } for (Cluster cluster : sessionParams.getClusters()) { - if (cluster.getEndpoints().isEmpty()) { + if (cluster.getEndpoints().isEmpty()) throw new IllegalArgumentException("Cannot feed to empty cluster."); - } } for (int i = 0; i < sessionParams.getClusters().size(); i++) { Cluster cluster = sessionParams.getClusters().get(i); - - clusters.add(new ClusterConnection( - this, - sessionParams.getFeedParams(), - sessionParams.getConnectionParams(), - sessionParams.getErrorReport(), - cluster, - i, - sessionParams.getClientQueueSize() / sessionParams.getClusters().size(), - timeoutExecutor)); - - } + clusters.add(new ClusterConnection(this, + sessionParams.getFeedParams(), + sessionParams.getConnectionParams(), + cluster, + i, + sessionParams.getClientQueueSize() / sessionParams.getClusters().size(), + timeoutExecutor)); + } operationStats = new OperationStats(sessionParams, clusters, incompleteResultsThrottler); maxRetries = sessionParams.getConnectionParams().getMaxRetries(); minTimeBetweenRetriesMs = sessionParams.getConnectionParams().getMinTimeBetweenRetriesMs(); @@ -122,21 +115,16 @@ public class OperationProcessor { } private boolean retriedThis(EndpointResult endpointResult, DocumentSendInfo documentSendInfo, int clusterId) { - final Result.Detail detail = endpointResult.getDetail(); - // If success, no retries to do. - if (detail.getResultType() == Result.ResultType.OPERATION_EXECUTED) { - return false; - } + Result.Detail detail = endpointResult.getDetail(); + if (detail.getResultType() == Result.ResultType.OPERATION_EXECUTED) return false; // Success: No retries int retries = documentSendInfo.incRetries(clusterId, detail); - if (retries > maxRetries) { - return false; - } + if (retries > maxRetries) return false; String exceptionMessage = detail.getException() == null ? "" : detail.getException().getMessage(); - if (exceptionMessage == null) { + if (exceptionMessage == null) exceptionMessage = ""; - } + // TODO: Return proper error code in structured data in next version of internal API. // Error codes from messagebus/src/cpp/messagebus/errorcode.h boolean retryThisOperation = @@ -151,12 +139,10 @@ public class OperationProcessor { if (retryThisOperation) { int waitTime = (int) (minTimeBetweenRetriesMs * (1 + random.nextDouble() / 3)); - log.finest("Retrying due to " + detail.toString() + " attempt " + retries - + " in " + waitTime + " ms."); - timeoutExecutor.schedule( - () -> postToCluster(clusters.get(clusterId), documentSendInfo.getDocument()), - waitTime, - TimeUnit.MILLISECONDS); + log.finest("Retrying due to " + detail.toString() + " attempt " + retries + " in " + waitTime + " ms."); + timeoutExecutor.schedule(() -> postToCluster(clusters.get(clusterId), documentSendInfo.getDocument()), + waitTime, + TimeUnit.MILLISECONDS); return true; } @@ -173,28 +159,20 @@ public class OperationProcessor { } DocumentSendInfo documentSendInfo = docSendInfoByOperationId.get(endpointResult.getOperationId()); - if (retriedThis(endpointResult, documentSendInfo, clusterId)) { - return null; - } + if (retriedThis(endpointResult, documentSendInfo, clusterId)) return null; - if (!documentSendInfo.addIfNotAlreadyThere(endpointResult.getDetail(), clusterId)) { - // Duplicate message, we have seen this operation before. - return null; - } + // Duplicate message + if ( ! documentSendInfo.addIfNotAlreadyThere(endpointResult.getDetail(), clusterId)) return null; // Is this the last operation we are waiting for? - if (documentSendInfo.detailCount() != numDestinations) { - return null; - } + if (documentSendInfo.detailCount() != numDestinations) return null; result = documentSendInfo.createResult(); docSendInfoByOperationId.remove(endpointResult.getOperationId()); String documentId = documentSendInfo.getDocument().getDocumentId(); - /** - * If we got a pending operation against this document - * dont't remove it from inflightDocuments and send blocked document operation - */ + // If we got a pending operation against this document + // dont't remove it from inflightDocuments and send blocked document operation List<Document> blockedDocuments = blockedDocumentsByDocumentId.get(documentId); if (blockedDocuments.isEmpty()) { inflightDocumentIds.remove(documentId); @@ -210,7 +188,6 @@ public class OperationProcessor { public void resultReceived(EndpointResult endpointResult, int clusterId) { Result result = process(endpointResult, clusterId); - if (result != null) { incompleteResultsThrottler.resultReady(result.isSuccess()); resultCallback.onCompletion(result.getDocumentId(), result); @@ -252,7 +229,6 @@ public class OperationProcessor { } private void sendToClusters(Document document) { - synchronized (monitor) { boolean traceThisDoc = traceEveryXOperation > 0 && traceCounter++ % traceEveryXOperation == 0; docSendInfoByOperationId.put(document.getOperationId(), new DocumentSendInfo(document, traceThisDoc)); @@ -319,4 +295,5 @@ public class OperationProcessor { throw new RuntimeException("Did not manage to shut down retry threads. Please report problem."); } } + } diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java index a2d5b18999e..388c71087ec 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.http.client; import com.yahoo.vespa.http.client.config.Cluster; import com.yahoo.vespa.http.client.config.ConnectionParams; import com.yahoo.vespa.http.client.config.Endpoint; +import com.yahoo.vespa.http.client.config.FeedParams; import com.yahoo.vespa.http.client.config.SessionParams; import org.junit.Test; @@ -36,7 +37,6 @@ public class SyncFeedClientTest { .build(); SyncFeedClient feedClient = new SyncFeedClient(sessionParams); - assertFeedSuccessful(feedClient); assertFeedSuccessful(feedClient); // ensure the client can be reused feedClient.close(); diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java index 3143282081b..5a4c6d05185 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java @@ -162,7 +162,7 @@ public class IOThreadTest { @Test public void requireThatEndpointConnectExceptionsArePropagated() - throws IOException, ServerResponseException, InterruptedException, TimeoutException, ExecutionException { + throws IOException, ServerResponseException, InterruptedException, TimeoutException, ExecutionException { when(apacheGatewayConnection.connect()).thenReturn(true); String errorMessage = "generic error message"; IOException cause = new IOException(errorMessage); diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 3b733105d2e..a16127931e9 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1099,6 +1099,8 @@ "public java.lang.Double setValue(java.lang.Double)", "public boolean equals(java.lang.Object)", "public int hashCode()", + "public java.lang.String toString(com.yahoo.tensor.TensorType)", + "public com.yahoo.tensor.Tensor$Cell detach()", "public bridge synthetic java.lang.Object setValue(java.lang.Object)", "public bridge synthetic java.lang.Object getValue()", "public bridge synthetic java.lang.Object getKey()" @@ -1180,6 +1182,8 @@ "public com.yahoo.tensor.Tensor sum()", "public com.yahoo.tensor.Tensor sum(java.lang.String)", "public com.yahoo.tensor.Tensor sum(java.util.List)", + "public java.util.List largest()", + "public java.util.List smallest()", "public abstract java.lang.String toString()", "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor)", "public static java.lang.String contentToString(com.yahoo.tensor.Tensor)", diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index 02f54b5790a..1da013de012 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -739,6 +739,11 @@ public abstract class IndexedTensor implements Tensor { @Override public Double getValue() { return value; } + @Override + public Cell detach() { + return new Cell(getKey(), value); + } + } // TODO: Make dimensionSizes a class diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index c2aa155d6bb..8b0aaa64551 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -63,7 +63,11 @@ public interface Tensor { /** Returns the value of a cell, or NaN if this cell does not exist/have no value */ double get(TensorAddress address); - /** Returns the cell of this in some undefined order */ + /** + * Returns the cell of this in some undefined order. + * A cell instances is only valid until next() is called. + * Call detach() on the cell to obtain a long-lived instance. + */ Iterator<Cell> cellIterator(); /** Returns the values of this in some undefined order */ @@ -250,6 +254,44 @@ public interface Tensor { default Tensor sum(String dimension) { return sum(Collections.singletonList(dimension)); } default Tensor sum(List<String> dimensions) { return reduce(Reduce.Aggregator.sum, dimensions); } + // ----------------- non-math query methods (that is, computations not returning a tensor) + + /** Returns the cell(s) of this tensor having the highest value */ + default List<Cell> largest() { + List<Cell> cells = new ArrayList<>(1); + double maxValue = Double.MIN_VALUE; + for (Iterator<Cell> i = cellIterator(); i.hasNext(); ) { + Cell cell = i.next(); + if (cell.getValue() > maxValue) { + cells.clear(); + cells.add(cell.detach()); + maxValue = cell.getDoubleValue(); + } + else if (cell.getValue() == maxValue) { + cells.add(cell.detach()); + } + } + return cells; + } + + /** Returns the cell(s) of this tensor having the lowest value */ + default List<Cell> smallest() { + List<Cell> cells = new ArrayList<>(1); + double minValue = Double.MAX_VALUE; + for (Iterator<Cell> i = cellIterator(); i.hasNext(); ) { + Cell cell = i.next(); + if (cell.getValue() < minValue) { + cells.clear(); + cells.add(cell.detach()); + minValue = cell.getDoubleValue(); + } + else if (cell.getValue() == minValue) { + cells.add(cell.detach()); + } + } + return cells; + } + // ----------------- serialization /** @@ -397,10 +439,10 @@ public interface Tensor { public Double getValue() { return value.doubleValue(); } /** Returns the value as a float */ - public float getFloatValue() { return value.floatValue(); } + public float getFloatValue() { return getValue().floatValue(); } /** Returns the value as a double */ - public double getDoubleValue() { return value.doubleValue(); } + public double getDoubleValue() { return getValue(); } @Override public Double setValue(Double value) { @@ -422,6 +464,13 @@ public interface Tensor { return getKey().hashCode() ^ getValue().hashCode(); // by Map.Entry spec } + public String toString(TensorType type) { return address.toString(type) + ":" + value; } + + /** + * Return a copy of this tensor cell which is valid beyond the lifetime of any iterator state which supplied it. + */ + public Cell detach() { return this; } + } interface Builder { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java index 4bfdb53e321..1a5bae9b02c 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java @@ -146,8 +146,10 @@ public class IndexedTensorTestCase { // Lookup from iterator Map<TensorAddress, Double> cellsOfIterator = new HashMap<>(); for (Iterator<Tensor.Cell> i = tensor.cellIterator(); i.hasNext(); ) { - Map.Entry<TensorAddress, Double> cell = i.next(); + Tensor.Cell cell = i.next(); cellsOfIterator.put(cell.getKey(), cell.getValue()); + assertEquals(cell.getValue(), cell.getDoubleValue(), 0.00001); + assertEquals(cell.getValue(), cell.getFloatValue(), 0.00001); } assertEquals(tensor.size(), cellsOfIterator.size()); for (int v = 0; v < vSize; v++) diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index 3d5d8d1f5ae..c6fbb9c009d 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -15,6 +15,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; import java.util.function.DoubleBinaryOperator; +import java.util.stream.Collectors; import static com.yahoo.tensor.TensorType.Dimension.Type; import static org.junit.Assert.assertEquals; @@ -248,6 +249,48 @@ public class TensorTestCase { Tensor.from("tensor(x{},y[3])", "{{x:0,y:0}:2,{x:0,y:1}:3}")); } + @Test + public void testLargest() { + assertLargest("{d1:l1,d2:l2}:6.0", + "tensor(d1{},d2{}):{{d1:l1,d2:l1}:5.0,{d1:l1,d2:l2}:6.0}"); + assertLargest("{d1:l1,d2:l1}:6.0, {d1:l1,d2:l2}:6.0", + "tensor(d1{},d2{}):{{d1:l1,d2:l1}:6.0,{d1:l1,d2:l2}:6.0}"); + assertLargest("{d1:l1,d2:l1}:6.0, {d1:l1,d2:l2}:6.0", + "tensor(d1{},d2{}):{{d1:l1,d2:l1}:6.0,{d1:l1,d2:l3}:5.0,{d1:l1,d2:l2}:6.0}"); + assertLargest("{x:1,y:1}:4.0", + "tensor(x[2],y[2]):[[1,2],[3,4]"); + assertLargest("{x:0,y:0}:4.0, {x:1,y:1}:4.0", + "tensor(x[2],y[2]):[[4,2],[3,4]"); + } + + @Test + public void testSmallest() { + assertSmallest("{d1:l1,d2:l1}:5.0", + "tensor(d1{},d2{}):{{d1:l1,d2:l1}:5.0,{d1:l1,d2:l2}:6.0}"); + assertSmallest("{d1:l1,d2:l1}:6.0, {d1:l1,d2:l2}:6.0", + "tensor(d1{},d2{}):{{d1:l1,d2:l1}:6.0,{d1:l1,d2:l2}:6.0}"); + assertSmallest("{d1:l1,d2:l1}:5.0, {d1:l1,d2:l2}:5.0", + "tensor(d1{},d2{}):{{d1:l1,d2:l1}:5.0,{d1:l1,d2:l3}:6.0,{d1:l1,d2:l2}:5.0}"); + assertSmallest("{x:0,y:0}:1.0", + "tensor(x[2],y[2]):[[1,2],[3,4]"); + assertSmallest("{x:0,y:1}:2.0", + "tensor(x[2],y[2]):[[4,2],[3,4]"); + } + + private void assertLargest(String expectedCells, String tensorString) { + Tensor tensor = Tensor.from(tensorString); + assertEquals(expectedCells, asString(tensor.largest(), tensor.type())); + } + + private void assertSmallest(String expectedCells, String tensorString) { + Tensor tensor = Tensor.from(tensorString); + assertEquals(expectedCells, asString(tensor.smallest(), tensor.type())); + } + + private String asString(List<Tensor.Cell> cells, TensorType type) { + return cells.stream().map(cell -> cell.toString(type)).collect(Collectors.joining(", ")); + } + private void assertTensorModify(DoubleBinaryOperator op, Tensor init, Tensor update, Tensor expected) { assertEquals(expected, init.modify(op, update.cells())); } |