diff options
| author | Lester Solbakken <lesters@users.noreply.github.com> | 2020-04-02 13:06:54 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-04-02 13:06:54 +0200 |
| commit | cd15c0e9021bce99839fcccb40d9406054de6c08 (patch) | |
| tree | 2d7c5a1418dcce1878de837d38272e0fadb33920 | |
| parent | c339a407bdcdd3a33e03628d593830dd66091072 (diff) | |
| parent | 448f5a09f05e0d17d58f9433d8fe78403f0ff78d (diff) | |
Merge branch 'master' into partial_update_bag_as_map_and_tensor
333 files changed, 5628 insertions, 2661 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index 53fcbbad9a2..2edf48103cf 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -3,6 +3,14 @@ include(${CMAKE_CURRENT_LIST_DIR}/vtag.cmake) +if (VESPA_USE_SANITIZER) + if (VESPA_USE_SANITIZER STREQUAL "address" OR VESPA_USE_SANITIZER STREQUAL "thread") + message("-- Instrumenting code using ${VESPA_USE_SANITIZER} sanitizer") + else() + message(FATAL_ERROR "Unsupported sanitizer option '${VESPA_USE_SANITIZER}'. Supported: 'address' or 'thread'") + endif() +endif() + # Build options # Whether to build unit tests as part of the 'all' target set(EXCLUDE_TESTS_FROM_ALL FALSE CACHE BOOL "If TRUE, do not build tests as part of the 'all' target") @@ -17,7 +25,13 @@ set(RUN_BENCHMARKS FALSE CACHE BOOL "If TRUE, benchmarks are run together with t set(AUTORUN_UNIT_TESTS FALSE CACHE BOOL "If TRUE, tests will be run immediately after linking the test executable") # Warnings -set(C_WARN_OPTS "-Winline -Wuninitialized -Werror -Wall -W -Wchar-subscripts -Wcomment -Wformat -Wparentheses -Wreturn-type -Wswitch -Wtrigraphs -Wunused -Wshadow -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings") +set(C_WARN_OPTS "-Wuninitialized -Werror -Wall -W -Wchar-subscripts -Wcomment -Wformat -Wparentheses -Wreturn-type -Wswitch -Wtrigraphs -Wunused -Wshadow -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings") +if (VESPA_USE_SANITIZER) + # Instrumenting code changes binary size, which triggers inlining warnings that + # don't happen during normal, non-instrumented compilation. +else() + set(C_WARN_OPTS "-Winline ${C_WARN_OPTS}") +endif() # Warnings that are specific to C++ compilation # Note: this is not a union of C_WARN_OPTS, since CMAKE_CXX_FLAGS already includes CMAKE_C_FLAGS, which in turn includes C_WARN_OPTS transitively @@ -46,7 +60,11 @@ else() endif() # C and C++ compiler flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DXXH_INLINE_ALL -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}") +# AddressSanitizer/ThreadSanitizer work for both GCC and Clang +if (VESPA_USE_SANITIZER) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=${VESPA_USE_SANITIZER}") +endif() if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} ${CXX_SPECIFIC_WARN_OPTS} -std=c++1z -fdiagnostics-color=auto ${EXTRA_CXX_FLAGS}") else() diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperTestServer.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperTestServer.java index 34c8fafa702..73b4163d542 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperTestServer.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperTestServer.java @@ -8,6 +8,7 @@ import org.apache.zookeeper.server.ZooKeeperServer; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; +import java.time.Duration; /** * This class sets up a zookeeper server, such that we can test fleetcontroller zookeeper parts without stubbing in the client. @@ -15,7 +16,7 @@ import java.net.InetSocketAddress; public class ZooKeeperTestServer { private File zooKeeperDir; private ZooKeeperServer server; - private static final int tickTime = 100; + private static final Duration tickTime = Duration.ofMillis(2000); private NIOServerCnxnFactory factory; private static final String DIR_PREFIX = "test_fltctrl_zk"; private static final String DIR_POSTFIX = "sdir"; @@ -31,7 +32,7 @@ public class ZooKeeperTestServer { throw new IllegalStateException("Failed to create directory " + zooKeeperDir); } zooKeeperDir.deleteOnExit(); - server = new ZooKeeperServer(zooKeeperDir, zooKeeperDir, tickTime); + server = new ZooKeeperServer(zooKeeperDir, zooKeeperDir, (int)tickTime.toMillis()); final int maxcc = 10000; // max number of connections from the same client factory = new NIOServerCnxnFactory(); factory.configure(new InetSocketAddress(port), maxcc); // Use any port diff --git a/config-lib/src/main/java/com/yahoo/config/FileReference.java b/config-lib/src/main/java/com/yahoo/config/FileReference.java index 3b95c2fbd4c..ee99ebfa2b7 100755 --- a/config-lib/src/main/java/com/yahoo/config/FileReference.java +++ b/config-lib/src/main/java/com/yahoo/config/FileReference.java @@ -28,14 +28,16 @@ public final class FileReference { } @Override - public int hashCode() { - return value.hashCode(); + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FileReference that = (FileReference) o; + return value.equals(that.value); } @Override - public boolean equals(Object other) { - return other instanceof FileReference && - value.equals(((FileReference)other).value); + public int hashCode() { + return Objects.hash(value); } @Override diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 0c061dd8222..4ccf34d30b0 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -606,521 +606,5 @@ "public static final com.yahoo.config.application.api.ValidationOverrides empty", "public static final com.yahoo.config.application.api.ValidationOverrides all" ] - }, - "com.yahoo.config.model.api.ApplicationInfo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.config.provision.ApplicationId, long, com.yahoo.config.model.api.Model)", - "public com.yahoo.config.provision.ApplicationId getApplicationId()", - "public long getGeneration()", - "public com.yahoo.config.model.api.Model getModel()", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ConfigChangeAction$Type": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.model.api.ConfigChangeAction$Type[] values()", - "public static com.yahoo.config.model.api.ConfigChangeAction$Type valueOf(java.lang.String)", - "public java.lang.String toString()" - ], - "fields": [ - "public static final enum com.yahoo.config.model.api.ConfigChangeAction$Type RESTART", - "public static final enum com.yahoo.config.model.api.ConfigChangeAction$Type REFEED" - ] - }, - "com.yahoo.config.model.api.ConfigChangeAction": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract com.yahoo.config.model.api.ConfigChangeAction$Type getType()", - "public abstract java.lang.String getMessage()", - "public abstract java.util.List getServices()", - "public abstract boolean allowed()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ConfigChangeRefeedAction": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.model.api.ConfigChangeAction" - ], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public com.yahoo.config.model.api.ConfigChangeAction$Type getType()", - "public java.lang.String name()", - "public abstract java.lang.String getDocumentType()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ConfigChangeRestartAction": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.model.api.ConfigChangeAction" - ], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public com.yahoo.config.model.api.ConfigChangeAction$Type getType()", - "public boolean allowed()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ConfigDefinitionRepo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract java.util.Map getConfigDefinitions()", - "public abstract com.yahoo.vespa.config.buildergen.ConfigDefinition get(com.yahoo.vespa.config.ConfigDefinitionKey)" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ConfigDefinitionStore": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract com.yahoo.vespa.config.ConfigDefinition getConfigDefinition(com.yahoo.vespa.config.ConfigDefinitionKey)" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ConfigModelPlugin": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [], - "fields": [] - }, - "com.yahoo.config.model.api.ConfigServerSpec": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract java.lang.String getHostName()", - "public abstract int getConfigServerPort()", - "public int getHttpPort()", - "public abstract int getZooKeeperPort()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ContainerEndpoint": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, java.util.List)", - "public java.lang.String clusterId()", - "public java.util.List names()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.EndpointCertificateMetadata": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, java.lang.String, int)", - "public java.lang.String keyName()", - "public java.lang.String certName()", - "public int version()", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.EndpointCertificateSecrets": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, java.lang.String)", - "public java.lang.String certificate()", - "public java.lang.String key()", - "public boolean isMissing()" - ], - "fields": [ - "public static final com.yahoo.config.model.api.EndpointCertificateSecrets MISSING" - ] - }, - "com.yahoo.config.model.api.FileDistribution": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void startDownload(java.lang.String, int, java.util.Set)", - "public abstract java.io.File getFileReferencesDir()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.HostInfo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, java.util.Collection)", - "public java.lang.String getHostname()", - "public java.util.Collection getServices()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.HostProvisioner": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract com.yahoo.config.provision.HostSpec allocateHost(java.lang.String)", - "public abstract java.util.List prepare(com.yahoo.config.provision.ClusterSpec, com.yahoo.config.provision.Capacity, int, com.yahoo.config.provision.ProvisionLogger)" - ], - "fields": [] - }, - "com.yahoo.config.model.api.Model": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract com.yahoo.vespa.config.ConfigPayload getConfig(com.yahoo.vespa.config.ConfigKey, com.yahoo.vespa.config.buildergen.ConfigDefinition)", - "public abstract java.util.Set allConfigsProduced()", - "public abstract java.util.Collection getHosts()", - "public abstract java.util.Set allConfigIds()", - "public abstract void distributeFiles(com.yahoo.config.model.api.FileDistribution)", - "public abstract java.util.Set fileReferences()", - "public abstract com.yahoo.config.provision.AllocatedHosts allocatedHosts()", - "public boolean allowModelVersionMismatch(java.time.Instant)", - "public boolean skipOldConfigModels(java.time.Instant)", - "public com.yahoo.component.Version version()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ModelContext$Properties": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract boolean multitenant()", - "public abstract com.yahoo.config.provision.ApplicationId applicationId()", - "public abstract java.util.List configServerSpecs()", - "public abstract com.yahoo.config.provision.HostName loadBalancerName()", - "public abstract java.net.URI ztsUrl()", - "public abstract java.lang.String athenzDnsSuffix()", - "public abstract boolean hostedVespa()", - "public abstract com.yahoo.config.provision.Zone zone()", - "public abstract java.util.Set endpoints()", - "public abstract boolean isBootstrap()", - "public abstract boolean isFirstTimeDeployment()", - "public boolean useDedicatedNodeForLogserver()", - "public abstract boolean useAdaptiveDispatch()", - "public java.util.Optional tlsSecrets()", - "public java.util.Optional endpointCertificateSecrets()", - "public abstract double defaultTermwiseLimit()", - "public abstract boolean useBucketSpaceMetric()", - "public boolean useNewAthenzFilter()", - "public boolean usePhraseSegmenting()", - "public java.lang.String proxyProtocol()", - "public java.util.Optional athenzDomain()", - "public boolean useDedicatedNodesWhenUnspecified()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ModelContext": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract com.yahoo.config.application.api.ApplicationPackage applicationPackage()", - "public abstract java.util.Optional previousModel()", - "public abstract java.util.Optional permanentApplicationPackage()", - "public abstract java.util.Optional hostProvisioner()", - "public abstract com.yahoo.config.application.api.DeployLogger deployLogger()", - "public abstract com.yahoo.config.model.api.ConfigDefinitionRepo configDefinitionRepo()", - "public abstract com.yahoo.config.application.api.FileRegistry getFileRegistry()", - "public abstract com.yahoo.config.model.api.ModelContext$Properties properties()", - "public java.util.Optional appDir()", - "public java.util.Optional wantedDockerImageRepository()", - "public abstract com.yahoo.component.Version modelVespaVersion()", - "public abstract com.yahoo.component.Version wantedNodeVespaVersion()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ModelCreateResult": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.config.model.api.Model, java.util.List)", - "public com.yahoo.config.model.api.Model getModel()", - "public java.util.List getConfigChangeActions()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ModelFactory": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract com.yahoo.component.Version version()", - "public abstract com.yahoo.config.model.api.Model createModel(com.yahoo.config.model.api.ModelContext)", - "public abstract com.yahoo.config.model.api.ModelCreateResult createAndValidateModel(com.yahoo.config.model.api.ModelContext, com.yahoo.config.model.api.ValidationParameters)" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ModelState": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract com.yahoo.config.model.api.Model getModel()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.PortInfo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(int, java.util.Collection)", - "public int getPort()", - "public java.util.Collection getTags()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.ServiceInfo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, java.lang.String, java.util.Collection, java.util.Map, java.lang.String, java.lang.String)", - "public java.lang.String getServiceName()", - "public java.lang.String getConfigId()", - "public java.lang.String getServiceType()", - "public java.util.Optional getProperty(java.lang.String)", - "public java.util.Collection getPorts()", - "public java.lang.String getHostName()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.SuperModel": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.util.Map, boolean)", - "public java.util.Map getModelsPerTenant()", - "public java.util.Map getModels()", - "public boolean isComplete()", - "public java.util.List getAllApplicationInfos()", - "public java.util.Optional getApplicationInfo(com.yahoo.config.provision.ApplicationId)", - "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo)", - "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)", - "public com.yahoo.config.model.api.SuperModel cloneAsComplete()", - "public java.util.Set getApplicationIds()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.SuperModelListener": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void applicationActivated(com.yahoo.config.model.api.SuperModel, com.yahoo.config.model.api.ApplicationInfo)", - "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)", - "public abstract void notifyOfCompleteness(com.yahoo.config.model.api.SuperModel)" - ], - "fields": [] - }, - "com.yahoo.config.model.api.SuperModelProvider": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void registerListener(com.yahoo.config.model.api.SuperModelListener)", - "public abstract com.yahoo.config.model.api.SuperModel getSuperModel()" - ], - "fields": [] - }, - "com.yahoo.config.model.api.TlsSecrets": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, java.lang.String)", - "public void <init>(com.yahoo.config.model.api.EndpointCertificateSecrets)", - "public java.lang.String certificate()", - "public java.lang.String key()", - "public boolean isMissing()" - ], - "fields": [ - "public static final com.yahoo.config.model.api.TlsSecrets MISSING" - ] - }, - "com.yahoo.config.model.api.ValidationParameters$CheckRouting": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.model.api.ValidationParameters$CheckRouting[] values()", - "public static com.yahoo.config.model.api.ValidationParameters$CheckRouting valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.config.model.api.ValidationParameters$CheckRouting TRUE", - "public static final enum com.yahoo.config.model.api.ValidationParameters$CheckRouting FALSE" - ] - }, - "com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange[] values()", - "public static com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange TRUE", - "public static final enum com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange FALSE" - ] - }, - "com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors[] values()", - "public static com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors TRUE", - "public static final enum com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors FALSE" - ] - }, - "com.yahoo.config.model.api.ValidationParameters": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors)", - "public void <init>(com.yahoo.config.model.api.ValidationParameters$CheckRouting)", - "public void <init>(com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors, com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange, com.yahoo.config.model.api.ValidationParameters$CheckRouting)", - "public boolean ignoreValidationErrors()", - "public boolean failOnIncompatibleChanges()", - "public boolean checkRouting()" - ], - "fields": [] } }
\ No newline at end of file diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java index e63e4ce3af6..96e76e46cda 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java @@ -10,8 +10,6 @@ public interface ConfigServerSpec { String getHostName(); int getConfigServerPort(); - // TODO: Remove when latest model version in use is 7.47 - default int getHttpPort() { return getConfigServerPort() + 1; } int getZooKeeperPort(); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java b/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java index bf58000dd36..4edf3c455d0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java @@ -18,15 +18,17 @@ public interface HostProvisioner { // TODO: Remove HostSpec allocateHost(String alias); + @Deprecated // TODO: Remove after April 2020 + List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger); + /** * Prepares allocation of a set of hosts with a given type, common id and the amount. * * @param cluster the cluster to allocate nodes to * @param capacity the capacity describing the capacity requested - * @param groups the number of groups to divide the nodes into * @param logger a logger to which messages to the deployer may be written * @return the specification of the allocated hosts */ - List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger); + List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java index a3478026520..689e2524dde 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi // Not really "public", only annotated as such to enable the ABI checker plugin package com.yahoo.config.model.api; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index e37b0b07746..896c6ea9a7f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -43,7 +43,6 @@ public class TestProperties implements ModelContext.Properties { private double defaultTermwiseLimit = 1.0; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private boolean useNewAthenzFilter = false; - private boolean useDedicatedNodesWhenUnspecified = false; private AthenzDomain athenzDomain; @Override public boolean multitenant() { return multitenant; } @@ -65,7 +64,7 @@ public class TestProperties implements ModelContext.Properties { @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useBucketSpaceMetric() { return true; } @Override public boolean useNewAthenzFilter() { return useNewAthenzFilter; } - @Override public boolean useDedicatedNodesWhenUnspecified() { return useDedicatedNodesWhenUnspecified; } + @Override public boolean useDedicatedNodesWhenUnspecified() { return true; } @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); } public TestProperties setDefaultTermwiseLimit(double limit) { @@ -118,11 +117,6 @@ public class TestProperties implements ModelContext.Properties { return this; } - public TestProperties setUseDedicatedNodesWhenUnspecified(boolean useDedicatedNodesWhenUnspecified) { - this.useDedicatedNodesWhenUnspecified = useDedicatedNodesWhenUnspecified; - return this; - } - public TestProperties setAthenzDomain(AthenzDomain domain) { this.athenzDomain = domain; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java index f909f3864da..201b69c1aae 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java @@ -45,10 +45,16 @@ public class HostsXmlProvisioner implements HostProvisioner { } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, int groups, ProvisionLogger logger) { throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported"); } + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, ProvisionLogger logger) { + throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported"); + } + private HostSpec host2HostSpec(Host host) { return new HostSpec(host.hostname(), host.aliases()); } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 6047b6a9818..298517b85f6 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -6,6 +6,7 @@ import com.yahoo.collections.Pair; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; @@ -57,6 +58,8 @@ public class InMemoryProvisioner implements HostProvisioner { /** Use this index as start index for all clusters */ private final int startIndexForClusters; + private final boolean useMaxResources; + /** Creates this with a number of nodes with resources 1, 3, 9, 1 */ public InMemoryProvisioner(int nodeCount) { this(nodeCount, defaultResources); @@ -64,27 +67,29 @@ public class InMemoryProvisioner implements HostProvisioner { /** Creates this with a number of nodes with given resources */ public InMemoryProvisioner(int nodeCount, NodeResources resources) { - this(Map.of(resources, createHostInstances(nodeCount)), true, 0); + this(Map.of(resources, createHostInstances(nodeCount)), true, false, 0); } /** Creates this with a set of host names of the flavor 'default' */ public InMemoryProvisioner(boolean failOnOutOfCapacity, String... hosts) { - this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, 0); + this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, false, 0); } /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) { - this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, 0, retiredHostNames); + this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, 0, retiredHostNames); } /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { - this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, startIndexForClusters, retiredHostNames); + this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, startIndexForClusters, retiredHostNames); } public InMemoryProvisioner(Map<NodeResources, Collection<Host>> hosts, boolean failOnOutOfCapacity, + boolean useMaxResources, int startIndexForClusters, String ... retiredHostNames) { this.failOnOutOfCapacity = failOnOutOfCapacity; + this.useMaxResources = useMaxResources; for (Map.Entry<NodeResources, Collection<Host>> hostsWithResources : hosts.entrySet()) for (Host host : hostsWithResources.getValue()) freeNodes.put(hostsWithResources.getKey(), host); @@ -113,34 +118,43 @@ public class InMemoryProvisioner implements HostProvisioner { } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) { - if (cluster.group().isPresent() && groups > 1) + return prepare(cluster, requestedCapacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { + if (useMaxResources) + return prepare(cluster, requested.maxResources(), requested.isRequired(), requested.canFail()); + else + return prepare(cluster, requested.minResources(), requested.isRequired(), requested.canFail()); + } + + public List<HostSpec> prepare(ClusterSpec cluster, ClusterResources requested, boolean required, boolean canFail) { + if (cluster.group().isPresent() && requested.groups() > 1) throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created"); - if (requestedCapacity.nodeCount() % groups != 0) - throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " + - groups + " groups, but the node count is not divisible into this number of groups"); - int capacity = failOnOutOfCapacity || requestedCapacity.isRequired() - ? requestedCapacity.nodeCount() - : Math.min(requestedCapacity.nodeCount(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster)); - if (groups > capacity) - groups = capacity; + int capacity = failOnOutOfCapacity || required + ? requested.nodes() + : Math.min(requested.nodes(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster)); + int groups = requested.groups() > capacity ? capacity : requested.groups(); List<HostSpec> allocation = new ArrayList<>(); if (groups == 1) { allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(0))), - requestedCapacity.nodeResources(), + requested.nodeResources(), capacity, startIndexForClusters, - requestedCapacity.canFail())); + canFail)); } else { for (int i = 0; i < groups; i++) { allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(i))), - requestedCapacity.nodeResources(), + requested.nodeResources(), capacity / groups, allocation.size(), - requestedCapacity.canFail())); + canFail)); } } for (ListIterator<HostSpec> i = allocation.listIterator(); i.hasNext(); ) { @@ -162,7 +176,7 @@ public class InMemoryProvisioner implements HostProvisioner { host.dockerImageRepo()); } - private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, Optional<NodeResources> requestedResources, + private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, NodeResources requestedResources, int nodesInGroup, int startIndex, boolean canFail) { List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>()); allocations.put(clusterGroup, allocation); @@ -170,8 +184,8 @@ public class InMemoryProvisioner implements HostProvisioner { // Check if the current allocations are compatible with the new request for (int i = allocation.size() - 1; i >= 0; i--) { Optional<NodeResources> currentResources = allocation.get(0).flavor().map(Flavor::resources); - if (currentResources.isEmpty() || requestedResources.isEmpty()) continue; - if (!currentResources.get().compatibleWith(requestedResources.get())) { + if (currentResources.isEmpty() || requestedResources == NodeResources.unspecified) continue; + if (!currentResources.get().compatibleWith(requestedResources)) { HostSpec removed = allocation.remove(i); freeNodes.put(currentResources.get(), new Host(removed.hostname())); // Return the node back to free pool } @@ -182,7 +196,7 @@ public class InMemoryProvisioner implements HostProvisioner { // Find the smallest host that can fit the requested requested Optional<NodeResources> hostResources = freeNodes.keySet().stream() .sorted(new MemoryDiskCpu()) - .filter(resources -> requestedResources.isEmpty() || resources.satisfies(requestedResources.get())) + .filter(resources -> requestedResources == NodeResources.unspecified || resources.satisfies(requestedResources)) .findFirst(); if (hostResources.isEmpty()) { if (canFail) @@ -195,8 +209,9 @@ public class InMemoryProvisioner implements HostProvisioner { if (freeNodes.get(hostResources.get()).isEmpty()) freeNodes.removeAll(hostResources.get()); ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++); allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(), - hostResources.map(Flavor::new), Optional.of(membership), - newHost.version(), Optional.empty(), requestedResources)); + hostResources.map(Flavor::new), Optional.of(membership), + newHost.version(), Optional.empty(), + requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources))); } nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex); diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index 180a16f3c8f..8945223447f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -30,6 +30,7 @@ public class SingleNodeProvisioner implements HostProvisioner { host = new Host(HostName.getLocalhost()); this.hostSpec = new HostSpec(host.hostname(), host.aliases()); } + public SingleNodeProvisioner(Flavor flavor) { host = new Host(HostName.getLocalhost()); this.hostSpec = new HostSpec(host.hostname(), host.aliases(), flavor); @@ -41,7 +42,14 @@ public class SingleNodeProvisioner implements HostProvisioner { } @Override - public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1 + @Deprecated // TODO: Remove after April 2020 + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + return prepare(cluster, capacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + // TODO: This should fail if capacity requested is more than 1 List<HostSpec> hosts = new ArrayList<>(); hosts.add(new HostSpec(host.hostname(), host.aliases(), ClusterMembership.from(cluster, counter++))); return hosts; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index 90f061d933d..aba6cf9a233 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -10,11 +10,11 @@ import java.io.Serializable; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashSet; +import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.Set; - /** * An index definition in a search definition. * Two indices are equal if they have the same name and the same settings, except @@ -24,6 +24,8 @@ import java.util.Set; */ public class Index implements Cloneable, Serializable { + public static enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES } + public enum Type { VESPA("vespa"); @@ -61,7 +63,9 @@ public class Index implements Cloneable, Serializable { /** The boolean index definition, if set */ private BooleanIndexDefinition boolIndex; - private Optional<HnswIndexParams> hnswIndexParams; + private Optional<HnswIndexParams> hnswIndexParams = Optional.empty(); + + private Optional<DistanceMetric> distanceMetric = Optional.empty(); /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */ private boolean interleavedFeatures = false; @@ -134,12 +138,13 @@ public class Index implements Cloneable, Serializable { stemming == index.stemming && type == index.type && Objects.equals(boolIndex, index.boolIndex) && + Objects.equals(distanceMetric, index.distanceMetric) && Objects.equals(hnswIndexParams, index.hnswIndexParams); } @Override public int hashCode() { - return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, hnswIndexParams, interleavedFeatures); + return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, distanceMetric, hnswIndexParams, interleavedFeatures); } public String toString() { @@ -187,6 +192,16 @@ public class Index implements Cloneable, Serializable { boolIndex = def; } + public Optional<DistanceMetric> getDistanceMetric() { + return distanceMetric; + } + + public void setDistanceMetric(String value) { + String upper = value.toUpperCase(Locale.ENGLISH); + DistanceMetric dm = DistanceMetric.valueOf(upper); + distanceMetric = Optional.of(dm); + } + public Optional<HnswIndexParams> getHnswIndexParams() { return hnswIndexParams; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 5b87fdcf5f6..8b5f7658475 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -240,13 +240,14 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce aaB.tensortype(attribute.tensorType().get().toString()); } aaB.imported(imported); + var dma = attribute.distanceMetric(); if (attribute.hnswIndexParams().isPresent()) { var ib = new AttributesConfig.Attribute.Index.Builder(); var params = attribute.hnswIndexParams().get(); ib.hnsw.enabled(true); ib.hnsw.maxlinkspernode(params.maxLinksPerNode()); ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert()); - var dm = AttributesConfig.Attribute.Index.Hnsw.Distancemetric.Enum.valueOf(params.distanceMetric().toString()); + var dm = AttributesConfig.Attribute.Index.Hnsw.Distancemetric.Enum.valueOf(dma.toString()); ib.hnsw.distancemetric(dm); aaB.index(ib); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index 9ed5e4ca2de..1661a80f238 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -24,6 +24,7 @@ import com.yahoo.document.datatypes.Float16FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.tensor.TensorType; +import static com.yahoo.searchdefinition.Index.DistanceMetric; import java.io.Serializable; import java.util.LinkedHashSet; @@ -66,7 +67,9 @@ public final class Attribute implements Cloneable, Serializable { /** This is set if the type of this is REFERENCE */ private final Optional<StructuredDataType> referenceDocumentType; - private Optional<HnswIndexParams> hnswIndexParams; + private Optional<DistanceMetric> distanceMetric = Optional.empty(); + + private Optional<HnswIndexParams> hnswIndexParams = Optional.empty(); private boolean isPosition = false; private final Sorting sorting = new Sorting(); @@ -152,7 +155,6 @@ public final class Attribute implements Cloneable, Serializable { setCollectionType(collectionType); this.tensorType = tensorType; this.referenceDocumentType = referenceDocumentType; - this.hnswIndexParams = Optional.empty(); } public Attribute convertToArray() { @@ -197,6 +199,11 @@ public final class Attribute implements Cloneable, Serializable { public double densePostingListThreshold() { return densePostingListThreshold; } public Optional<TensorType> tensorType() { return tensorType; } public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; } + + public static final DistanceMetric DEFAULT_DISTANCE_METRIC = DistanceMetric.EUCLIDEAN; + public DistanceMetric distanceMetric() { + return distanceMetric.orElse(DEFAULT_DISTANCE_METRIC); + } public Optional<HnswIndexParams> hnswIndexParams() { return hnswIndexParams; } public Sorting getSorting() { return sorting; } @@ -221,6 +228,7 @@ public final class Attribute implements Cloneable, Serializable { public void setUpperBound(long upperBound) { this.upperBound = upperBound; } public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; } public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } + public void setDistanceMetric(Optional<DistanceMetric> dm) { this.distanceMetric = dm; } public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } public String getName() { return name; } @@ -354,8 +362,8 @@ public final class Attribute implements Cloneable, Serializable { /** Returns whether these attributes describes the same entity, even if they have different names */ public boolean isCompatible(Attribute other) { - if ( ! this.type.equals(other.type)) return false; - if ( ! this.collectionType.equals(other.collectionType)) return false; + if (! this.type.equals(other.type)) return false; + if (! this.collectionType.equals(other.collectionType)) return false; if (this.isPrefetch() != other.isPrefetch()) return false; if (this.removeIfZero != other.removeIfZero) return false; if (this.createIfNonExistent != other.createIfNonExistent) return false; @@ -364,10 +372,11 @@ public final class Attribute implements Cloneable, Serializable { // if (this.noSearch != other.noSearch) return false; No backend consequences so compatible for now if (this.fastSearch != other.fastSearch) return false; if (this.huge != other.huge) return false; - if ( ! this.sorting.equals(other.sorting)) return false; - if (!this.tensorType.equals(other.tensorType)) return false; - if (!this.referenceDocumentType.equals(other.referenceDocumentType)) return false; - if (!this.hnswIndexParams.equals(other.hnswIndexParams)) return false; + if (! this.sorting.equals(other.sorting)) return false; + if (! Objects.equals(tensorType, other.tensorType)) return false; + if (! Objects.equals(referenceDocumentType, other.referenceDocumentType)) return false; + if (! Objects.equals(distanceMetric, other.distanceMetric)) return false; + if (! Objects.equals(hnswIndexParams, other.hnswIndexParams)) return false; return true; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java index 01434be8785..2f084d3e513 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java @@ -13,18 +13,13 @@ public class HnswIndexParams { public static final int DEFAULT_MAX_LINKS_PER_NODE = 16; public static final int DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT = 200; - public static final DistanceMetric DEFAULT_DISTANCE_METRIC = DistanceMetric.EUCLIDEAN; private final Optional<Integer> maxLinksPerNode; private final Optional<Integer> neighborsToExploreAtInsert; - private final Optional<DistanceMetric> distanceMetric; - - public static enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES } public static class Builder { private Optional<Integer> maxLinksPerNode = Optional.empty(); private Optional<Integer> neighborsToExploreAtInsert = Optional.empty(); - private Optional<DistanceMetric> distanceMetric = Optional.empty(); public void setMaxLinksPerNode(int value) { maxLinksPerNode = Optional.of(value); @@ -32,38 +27,31 @@ public class HnswIndexParams { public void setNeighborsToExploreAtInsert(int value) { neighborsToExploreAtInsert = Optional.of(value); } - public void setDistanceMetric(String value) { - String upper = value.toUpperCase(Locale.ENGLISH); - DistanceMetric dm = DistanceMetric.valueOf(upper); - distanceMetric = Optional.of(dm); - } public HnswIndexParams build() { - return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert, distanceMetric); + return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert); } } public HnswIndexParams() { this.maxLinksPerNode = Optional.empty(); this.neighborsToExploreAtInsert = Optional.empty(); - this.distanceMetric = Optional.empty(); } public HnswIndexParams(Optional<Integer> maxLinksPerNode, - Optional<Integer> neighborsToExploreAtInsert, - Optional<DistanceMetric> distanceMetric) { + Optional<Integer> neighborsToExploreAtInsert) { this.maxLinksPerNode = maxLinksPerNode; this.neighborsToExploreAtInsert = neighborsToExploreAtInsert; - this.distanceMetric = distanceMetric; } /** * Creates a new instance where values from the given parameter instance are used where they are present, * otherwise we use values from this. */ - public HnswIndexParams overrideFrom(HnswIndexParams rhs) { + public HnswIndexParams overrideFrom(Optional<HnswIndexParams> other) { + if (! other.isPresent()) return this; + HnswIndexParams rhs = other.get(); return new HnswIndexParams(rhs.maxLinksPerNode.or(() -> maxLinksPerNode), - rhs.neighborsToExploreAtInsert.or(() -> neighborsToExploreAtInsert), - rhs.distanceMetric.or(() -> distanceMetric)); + rhs.neighborsToExploreAtInsert.or(() -> neighborsToExploreAtInsert)); } public int maxLinksPerNode() { @@ -73,8 +61,4 @@ public class HnswIndexParams { public int neighborsToExploreAtInsert() { return neighborsToExploreAtInsert.orElse(DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT); } - - public DistanceMetric distanceMetric() { - return distanceMetric.orElse(DEFAULT_DISTANCE_METRIC); - } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java index 7f9da28b9ca..0c1f443dee3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java @@ -32,6 +32,7 @@ public class IndexOperation implements FieldOperation { private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); private Optional<Boolean> enableBm25 = Optional.empty(); + private Optional<String> distanceMetric = Optional.empty(); private Optional<HnswIndexParams.Builder> hnswIndexParams = Optional.empty(); public String getIndexName() { @@ -94,6 +95,9 @@ public class IndexOperation implements FieldOperation { if (enableBm25.isPresent()) { index.setInterleavedFeatures(enableBm25.get()); } + if (distanceMetric.isPresent()) { + index.setDistanceMetric(distanceMetric.get()); + } if (hnswIndexParams.isPresent()) { index.setHnswIndexParams(hnswIndexParams.get().build()); } @@ -127,6 +131,10 @@ public class IndexOperation implements FieldOperation { enableBm25 = Optional.of(value); } + public void setDistanceMetric(String value) { + this.distanceMetric = Optional.of(value); + } + public void setHnswIndexParams(HnswIndexParams.Builder params) { this.hnswIndexParams = Optional.of(params); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 2790f2ddf6e..c97ee2bd935 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -81,8 +81,9 @@ public class TensorFieldProcessor extends Processor { var index = field.getIndex(field.getName()); // TODO: Calculate default params based on tensor dimension size var params = new HnswIndexParams(); - if (index != null && index.getHnswIndexParams().isPresent()) { - params = params.overrideFrom(index.getHnswIndexParams().get()); + if (index != null) { + params = params.overrideFrom(index.getHnswIndexParams()); + field.getAttribute().setDistanceMetric(index.getDistanceMetric()); } field.getAttribute().setHnswIndexParams(params); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 3765e683b18..557a61ec211 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -111,8 +111,8 @@ public class HostSystem extends AbstractConfigProducer<Host> { } } - public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, int groups, DeployLogger logger) { - List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, groups, new ProvisionDeployLogger(logger)); + public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, DeployLogger logger) { + List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, new ProvisionDeployLogger(logger)); // TODO: Even if HostResource owns a set of memberships, we need to return a map because the caller needs the current membership. Map<HostResource, ClusterMembership> retAllocatedHosts = new LinkedHashMap<>(); for (HostSpec spec : allocatedHosts) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 001e6a9a407..09128e741b9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -301,6 +301,14 @@ public class VespaMetricSet { return metrics; } + private static void addSearchNodeExecutorMetrics(Set<Metric> metrics, String prefix) { + metrics.add(new Metric(prefix + ".queuesize.max")); + metrics.add(new Metric(prefix + ".queuesize.sum")); + metrics.add(new Metric(prefix + ".queuesize.count")); + metrics.add(new Metric(prefix + ".maxpending.last")); // TODO: Remove in Vespa 8 + metrics.add(new Metric(prefix + ".accepted.rate")); + } + private static Set<Metric> getSearchNodeMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); @@ -345,18 +353,12 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.search_protocol.docsum.requested_documents.count")); // Executors shared between all document dbs - metrics.add(new Metric("content.proton.executor.proton.maxpending.last")); - metrics.add(new Metric("content.proton.executor.proton.accepted.rate")); - metrics.add(new Metric("content.proton.executor.flush.maxpending.last")); - metrics.add(new Metric("content.proton.executor.flush.accepted.rate")); - metrics.add(new Metric("content.proton.executor.match.maxpending.last")); - metrics.add(new Metric("content.proton.executor.match.accepted.rate")); - metrics.add(new Metric("content.proton.executor.docsum.maxpending.last")); - metrics.add(new Metric("content.proton.executor.docsum.accepted.rate")); - metrics.add(new Metric("content.proton.executor.shared.maxpending.last")); - metrics.add(new Metric("content.proton.executor.shared.accepted.rate")); - metrics.add(new Metric("content.proton.executor.warmup.maxpending.last")); - metrics.add(new Metric("content.proton.executor.warmup.accepted.rate")); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.proton"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.flush"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.match"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.docsum"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.shared"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.warmup"); // jobs metrics.add(new Metric("content.proton.documentdb.job.total.average")); @@ -370,18 +372,12 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.job.removed_documents_prune.average")); // Threading service (per document db) - metrics.add(new Metric("content.proton.documentdb.threading_service.master.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.master.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.summary.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.summary.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_inverter.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_inverter.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_writer.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_writer.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.attribute_field_writer.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.attribute_field_writer.accepted.rate")); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.master"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.summary"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index_field_inverter"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index_field_writer"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.attribute_field_writer"); // lid space metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_bloat_factor.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index b4246171277..b6f7ab4ff62 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -138,7 +138,7 @@ public class RankSetupValidator extends Validator { private boolean execValidate(String configId, SearchCluster sc, String sdName, DeployLogger deployLogger) { String job = String.format("%s %s", binaryName, configId); - ProcessExecuter executer = new ProcessExecuter(); + ProcessExecuter executer = new ProcessExecuter(true); try { Pair<Integer, String> ret = executer.exec(job); if (ret.getFirst() != 0) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 5629956e8b9..804a5442608 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -65,12 +65,12 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { createSlobroks(deployLogger, admin, allocateHosts(admin.hostSystem(), "slobroks", nodesSpecification)); } else { - createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.count(), 2)); + createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.minResources().nodes(), 2)); } } private void assignLogserver(DeployState deployState, NodesSpecification nodesSpecification, Admin admin) { - if (nodesSpecification.count() > 1) throw new IllegalArgumentException("You can only request a single log server"); + if (nodesSpecification.minResources().nodes() > 1) throw new IllegalArgumentException("You can only request a single log server"); if (deployState.getProperties().applicationId().instance().isTester()) return; // No logserver is needed on tester applications if (nodesSpecification.isDedicated()) { Collection<HostResource> hosts = allocateHosts(admin.hostSystem(), "logserver", nodesSpecification); @@ -79,7 +79,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { Logserver logserver = createLogserver(deployState.getDeployLogger(), admin, hosts); createContainerOnLogserverHost(deployState, admin, logserver.getHostResource()); } else if (containerModels.iterator().hasNext()) { - List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.count(), false); + List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.minResources().nodes(), false); if (hosts.isEmpty()) return; // No log server can be created (and none is needed) createLogserver(deployState.getDeployLogger(), admin, hosts); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java index d34a11abdf4..80c95ad6b59 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java @@ -172,7 +172,12 @@ public class ModelElement { /** Returns the content of the attribute with the given name, or null if none */ public String stringAttribute(String name) { - if ( ! xml.hasAttribute(name)) return null; + return stringAttribute(name, null); + } + + /** Returns the content of the attribute with the given name, or the default value if none */ + public String stringAttribute(String name, String defaultValue) { + if ( ! xml.hasAttribute(name)) return defaultValue; return xml.getAttribute(name); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 384d891003e..6a52ff4f051 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -1,11 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.text.XML; @@ -18,6 +20,8 @@ import org.w3c.dom.Node; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; +import java.util.regex.Pattern; /** * A common utility class to represent a requirement for nodes during model building. @@ -27,11 +31,9 @@ import java.util.Optional; */ public class NodesSpecification { - private final boolean dedicated; - - private final int count; + private final ClusterResources min, max; - private final int groups; + private final boolean dedicated; /** The Vespa version we want the nodes to run */ private Version version; @@ -46,43 +48,51 @@ public class NodesSpecification { private final boolean exclusive; - /** The resources each node should have, or empty to use the default */ - private final Optional<NodeResources> resources; - /** The repo part of a docker image (without tag), optional */ private final Optional<String> dockerImageRepo; /** The ID of the cluster referencing this node specification, if any */ private final Optional<String> combinedId; - private NodesSpecification(boolean dedicated, int count, int groups, Version version, + private NodesSpecification(ClusterResources min, + ClusterResources max, + boolean dedicated, Version version, boolean required, boolean canFail, boolean exclusive, - Optional<NodeResources> resources, Optional<String> dockerImageRepo, + Optional<String> dockerImageRepo, Optional<String> combinedId) { + this.min = min; + this.max = max; this.dedicated = dedicated; - this.count = count; - this.groups = groups; this.version = version; this.required = required; this.canFail = canFail; this.exclusive = exclusive; - this.resources = resources; this.dockerImageRepo = dockerImageRepo; this.combinedId = combinedId; } - private NodesSpecification(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement, - Optional<String> combinedId, Optional<String> dockerImageRepo) { - this(dedicated, - nodesElement.integerAttribute("count", 1), - nodesElement.integerAttribute("groups", 1), - version, - nodesElement.booleanAttribute("required", false), - canFail, - nodesElement.booleanAttribute("exclusive", false), - getResources(nodesElement), - dockerImageToUse(nodesElement, dockerImageRepo), - combinedId); + private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, + ModelElement nodesElement, Optional<String> dockerImageRepo) { + var resolvedElement = resolveElement(nodesElement); + var combinedId = findCombinedId(nodesElement, resolvedElement); + var resources = toResources(resolvedElement); + return new NodesSpecification(resources.getFirst(), + resources.getSecond(), + dedicated, + version, + resolvedElement.booleanAttribute("required", false), + canFail, + resolvedElement.booleanAttribute("exclusive", false), + dockerImageToUse(resolvedElement, dockerImageRepo), + combinedId); + } + + private static Pair<ClusterResources, ClusterResources> toResources(ModelElement nodesElement) { + Pair<Integer, Integer> nodes = toRange(nodesElement.stringAttribute("count"), 1, Integer::parseInt); + Pair<Integer, Integer> groups = toRange(nodesElement.stringAttribute("groups"), 1, Integer::parseInt); + var min = new ClusterResources(nodes.getFirst(), groups.getFirst(), nodeResources(nodesElement).getFirst()); + var max = new ClusterResources(nodes.getSecond(), groups.getSecond(), nodeResources(nodesElement).getSecond()); + return new Pair<>(min, max); } /** Returns the ID of the cluster referencing this node specification, if any */ @@ -95,13 +105,6 @@ public class NodesSpecification { return containerIdReferencing(nodesElement); } - private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, - ModelElement nodesElement, Optional<String> dockerImage) { - var resolvedElement = resolveElement(nodesElement); - var combinedId = findCombinedId(nodesElement, resolvedElement); - return new NodesSpecification(dedicated, canFail, version, resolvedElement, combinedId, dockerImage); - } - /** Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element */ public static NodesSpecification from(ModelElement nodesElement, ConfigModelContext context) { return create(true, @@ -144,32 +147,33 @@ public class NodesSpecification { * Returns a requirement from <code>count</code> non-dedicated nodes in one group */ public static NodesSpecification nonDedicated(int count, ConfigModelContext context) { - return new NodesSpecification(false, - count, - 1, + return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified), + new ClusterResources(count, 1, NodeResources.unspecified), + false, context.getDeployState().getWantedNodeVespaVersion(), false, ! context.getDeployState().getProperties().isBootstrap(), false, - Optional.empty(), context.getDeployState().getWantedDockerImageRepo(), Optional.empty()); } /** Returns a requirement from <code>count</code> dedicated nodes in one group */ public static NodesSpecification dedicated(int count, ConfigModelContext context) { - return new NodesSpecification(true, - count, - 1, + return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified), + new ClusterResources(count, 1, NodeResources.unspecified), + true, context.getDeployState().getWantedNodeVespaVersion(), false, ! context.getDeployState().getProperties().isBootstrap(), false, - Optional.empty(), context.getDeployState().getWantedDockerImageRepo(), Optional.empty()); } + public ClusterResources minResources() { return min; } + public ClusterResources maxResources() { return max; } + /** * Returns whether this requires dedicated nodes. * Otherwise the model encountering this request should reuse nodes requested for other purposes whenever possible. @@ -183,12 +187,6 @@ public class NodesSpecification { */ public boolean isExclusive() { return exclusive; } - /** Returns the number of nodes required */ - public int count() { return count; } - - /** Returns the number of host groups this specifies. Default is 1 */ - public int groups() { return groups; } - public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, @@ -201,29 +199,36 @@ public class NodesSpecification { .combinedId(combinedId.map(ClusterSpec.Id::from)) .dockerImageRepo(dockerImageRepo) .build(); - return hostSystem.allocateHosts(cluster, Capacity.fromCount(count, resources, required, canFail), groups, logger); + return hostSystem.allocateHosts(cluster, Capacity.from(min, max, required, canFail), logger); } - private static Optional<NodeResources> getResources(ModelElement nodesElement) { + private static Pair<NodeResources, NodeResources> nodeResources(ModelElement nodesElement) { ModelElement resources = nodesElement.child("resources"); if (resources != null) { - return Optional.of(new NodeResources(resources.requiredDoubleAttribute("vcpu"), - parseGbAmount(resources.requiredStringAttribute("memory"), "B"), - parseGbAmount(resources.requiredStringAttribute("disk"), "B"), - Optional.ofNullable(resources.stringAttribute("bandwidth")) - .map(b -> parseGbAmount(b, "BPS")) - .orElse(0.3), - parseOptionalDiskSpeed(resources.stringAttribute("disk-speed")), - parseOptionalStorageType(resources.stringAttribute("storage-type")))); + return nodeResourcesFromResorcesElement(resources); } else if (nodesElement.stringAttribute("flavor") != null) { // legacy fallback - return Optional.of(NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor"))); + var flavorResources = NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor")); + return new Pair<>(flavorResources, flavorResources); } - else { // Get the default - return Optional.empty(); + else { + return new Pair<>(NodeResources.unspecified, NodeResources.unspecified); } } + private static Pair<NodeResources, NodeResources> nodeResourcesFromResorcesElement(ModelElement element) { + Pair<Double, Double> vcpu = toRange(element.requiredStringAttribute("vcpu"), .0, Double::parseDouble); + Pair<Double, Double> memory = toRange(element.requiredStringAttribute("memory"), .0, s -> parseGbAmount(s, "B")); + Pair<Double, Double> disk = toRange(element.requiredStringAttribute("disk"), .0, s -> parseGbAmount(s, "B")); + Pair<Double, Double> bandwith = toRange(element.stringAttribute("bandwith"), .3, s -> parseGbAmount(s, "BPS")); + NodeResources.DiskSpeed diskSpeed = parseOptionalDiskSpeed(element.stringAttribute("disk-speed")); + NodeResources.StorageType storageType = parseOptionalStorageType(element.stringAttribute("storage-type")); + + var min = new NodeResources(vcpu.getFirst(), memory.getFirst(), disk.getFirst(), bandwith.getFirst(), diskSpeed, storageType); + var max = new NodeResources(vcpu.getSecond(), memory.getSecond(), disk.getSecond(), bandwith.getSecond(), diskSpeed, storageType); + return new Pair<>(min, max); + } + private static double parseGbAmount(String byteAmount, String unit) { byteAmount = byteAmount.strip(); byteAmount = byteAmount.toUpperCase(); @@ -358,11 +363,28 @@ public class NodesSpecification { return dockerImageFromElement == null ? dockerImage : Optional.of(dockerImageFromElement); } + /** Parses a value ("value") or value range ("[min-value, max-value]") */ + private static <T> Pair<T, T> toRange(String s, T defaultValue, Function<String, T> valueParser) { + try { + if (s == null) return new Pair<>(defaultValue, defaultValue); + s = s.trim(); + if (s.startsWith("[") && s.endsWith("]")) { + String[] numbers = s.substring(1, s.length() - 1).split(","); + if (numbers.length != 2) throw new IllegalArgumentException(); + return new Pair<>(valueParser.apply(numbers[0].trim()), valueParser.apply(numbers[1].trim())); + } else { + return new Pair<>(valueParser.apply(s), valueParser.apply(s)); + } + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Expected a number or range on the form [min, max], but got '" + s + "'", e); + } + } + @Override public String toString() { - return "specification of " + count + (dedicated ? " dedicated " : " ") + "nodes" + - (resources.map(nodeResources -> " with resources " + nodeResources).orElse("")) + - (groups > 1 ? " in " + groups + " groups" : ""); + return "specification of " + (dedicated ? "dedicated " : "") + + (min.equals(max) ? min : "min " + min + " max " + max); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 97b78e1b9b1..c9caca1831f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -141,8 +141,7 @@ public class VespaDomBuilder extends VespaModelBuilder { } private void initializeService(AbstractService t, DeployState deployState, - HostSystem hostSystem, Element producerSpec) - { + HostSystem hostSystem, Element producerSpec) { initializeProducer(t, deployState, producerSpec); if (producerSpec != null) { if (producerSpec.hasAttribute(JVM_OPTIONS)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 7e2d6680827..31c8724d634 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -140,7 +140,7 @@ public abstract class Container extends AbstractService implements if (http == null) { return defaultHttpServer; } else { - return http.getHttpServer(); + return http.getHttpServer().orElse(null); } } @@ -228,10 +228,10 @@ public abstract class Container extends AbstractService implements // XXX unused - remove: from.allocatePort("http/1"); portsMeta.on(offset++).tag("http").tag("external"); - } else if (getHttp().getHttpServer() == null) { + } else if (getHttp().getHttpServer().isEmpty()) { // no http server ports } else { - for (ConnectorFactory connectorFactory : getHttp().getHttpServer().getConnectorFactories()) { + for (ConnectorFactory connectorFactory : getHttp().getHttpServer().get().getConnectorFactories()) { int port = getPort(connectorFactory); String name = "http/" + connectorFactory.getName(); from.requirePort(port, name); @@ -280,7 +280,7 @@ public abstract class Container extends AbstractService implements final Http http = getHttp(); if (http != null) { // TODO: allow the user to specify health port manually - if (http.getHttpServer() == null) { + if (http.getHttpServer().isEmpty()) { return -1; } else { return getRelativePort(0); @@ -303,7 +303,7 @@ public abstract class Container extends AbstractService implements .slobrokId(serviceSlobrokId())). filedistributor(filedistributorConfig()); if (clusterName != null) { - builder.discriminator(clusterName+"."+name); + builder.discriminator(clusterName + "." + name); } else { builder.discriminator(name); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java index ee61b34987a..3d9a1b2e665 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java @@ -10,6 +10,7 @@ import com.yahoo.osgi.provider.model.ComponentModel; * @author Tony Vaagenes */ public class FileStatusHandlerComponent extends Handler implements VipStatusConfig.Producer { + public static final String CLASS = "com.yahoo.container.handler.VipStatusHandler"; private final String fileName; @@ -26,4 +27,5 @@ public class FileStatusHandlerComponent extends Handler implements VipStatusConf builder.accessdisk(true). statusfile(fileName); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index f3758def2b1..470b82496a3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.container.StatisticsConfig; +import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.net.HostName; import com.yahoo.vespa.defaults.Defaults; @@ -29,7 +30,8 @@ public class ConfigserverCluster extends AbstractConfigProducer ZookeeperServerConfig.Producer, ConfigserverConfig.Producer, StatisticsConfig.Producer, - HealthMonitorConfig.Producer { + HealthMonitorConfig.Producer, + VipStatusConfig.Producer { private final CloudConfigOptions options; private ContainerCluster containerCluster; @@ -116,8 +118,8 @@ public class ConfigserverCluster extends AbstractConfigProducer } builder.serverId(HostName.getLocalhost()); - if (!containerCluster.getHttp().getHttpServer().getConnectorFactories().isEmpty()) { - builder.httpport(containerCluster.getHttp().getHttpServer().getConnectorFactories().get(0).getListenPort()); + if (!containerCluster.getHttp().getHttpServer().get().getConnectorFactories().isEmpty()) { + builder.httpport(containerCluster.getHttp().getHttpServer().get().getConnectorFactories().get(0).getListenPort()); } if (options.useVespaVersionInRequest().isPresent()) { builder.useVespaVersionInRequest(options.useVespaVersionInRequest().get()); @@ -178,4 +180,8 @@ public class ConfigserverCluster extends AbstractConfigProducer builder.snapshot_interval(60.0); } + @Override + public void getConfig(VipStatusConfig.Builder builder) { + builder.initiallyInRotation(false); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java index 400ddf80cf9..0fcf7b2d06c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -8,46 +8,39 @@ import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.component.chain.ChainedComponent; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; /** * Represents the http servers and filters of a container cluster. * * @author Tony Vaagenes + * @author bjorncs */ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> implements ServerConfig.Producer { - private FilterChains filterChains; - private JettyHttpServer httpServer; - private List<Binding> bindings; - private final Optional<AccessControl> accessControl; + private final FilterChains filterChains; + private final List<Binding> bindings = new CopyOnWriteArrayList<>(); + private volatile JettyHttpServer httpServer; + private volatile AccessControl accessControl; - public Http(List<Binding> bindings) { - this(bindings, null); + public Http(FilterChains chains) { + super("http"); + this.filterChains = chains; } - public Http(List<Binding> bindings, AccessControl accessControl) { - super( "http"); - this.bindings = Collections.unmodifiableList(bindings); - this.accessControl = Optional.ofNullable(accessControl); - } - - public void setFilterChains(FilterChains filterChains) { - this.filterChains = filterChains; - } - - public void setBindings(List<Binding> bindings) { - this.bindings = Collections.unmodifiableList(bindings); + public void setAccessControl(AccessControl accessControl) { + if (this.accessControl != null) throw new IllegalStateException("Access control already assigned"); + this.accessControl = accessControl; } public FilterChains getFilterChains() { return filterChains; } - public JettyHttpServer getHttpServer() { - return httpServer; + public Optional<JettyHttpServer> getHttpServer() { + return Optional.ofNullable(httpServer); } public void setHttpServer(JettyHttpServer newServer) { @@ -76,25 +69,21 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl } public Optional<AccessControl> getAccessControl() { - return accessControl; + return Optional.ofNullable(accessControl); } @Override public void getConfig(ServerConfig.Builder builder) { for (Binding binding : bindings) { builder.filter(new ServerConfig.Filter.Builder() - .id(binding.filterId().stringValue()) - .binding(binding.binding())); + .id(binding.filterId().stringValue()) + .binding(binding.binding())); } } @Override public void validate() { - validate(bindings); - } - - void validate(Collection<Binding> bindings) { - if (bindings.isEmpty()) return; + if (((Collection<Binding>) bindings).isEmpty()) return; if (filterChains == null) throw new IllegalArgumentException("Null FilterChains are not allowed when there are filter bindings"); @@ -107,5 +96,4 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl throw new RuntimeException("Can't find filter " + binding.filterId() + " for binding " + binding.binding()); } } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index f61618c789b..fb8e9dffbbb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -7,6 +7,7 @@ import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; +import java.time.Duration; import java.util.List; /** @@ -67,10 +68,13 @@ public class HostedSslConnectorFactory extends ConnectorFactory { @Override public void getConfig(ConnectorConfig.Builder connectorBuilder) { super.getConfig(connectorBuilder); - connectorBuilder.tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder() - .pathWhitelist(INSECURE_WHITELISTED_PATHS) - .enable(enforceClientAuth)); - connectorBuilder.proxyProtocol(configureProxyProtocol()); + connectorBuilder + .tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder() + .pathWhitelist(INSECURE_WHITELISTED_PATHS) + .enable(enforceClientAuth)) + .proxyProtocol(configureProxyProtocol()) + .idleTimeout(Duration.ofMinutes(3).toSeconds()) + .maxConnectionLife(Duration.ofMinutes(10).toSeconds()); } private ConnectorConfig.ProxyProtocol.Builder configureProxyProtocol() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index 61a588fb716..bfde9b9add1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -54,11 +54,10 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> filterChains = new FilterChainsBuilder().newChainsInstance(ancestor); } - Http http = new Http(bindings, accessControl); - http.setFilterChains(filterChains); - - buildHttpServers(deployState, ancestor, http, spec); - + Http http = new Http(filterChains); + http.getBindings().addAll(bindings); + http.setAccessControl(accessControl); + http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec)); return http; } @@ -131,10 +130,6 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> return result; } - private void buildHttpServers(DeployState deployState, AbstractConfigProducer ancestor, Http http, Element spec) { - http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec)); - } - static int readPort(ModelElement spec, boolean isHosted, DeployLogger logger) { Integer port = spec.integerAttribute("port"); if (port == null) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index c1f793e255d..06707786136 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -23,9 +23,11 @@ import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; @@ -324,13 +326,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.setHttp(buildHttp(deployState, cluster, httpElement)); } if (isHostedTenantApplication(context)) { - addHostedImplicitHttpIfNotPresent(deployState, cluster); + addHostedImplicitHttpIfNotPresent(cluster); + addHostedImplicitAccessControlIfNotPresent(deployState, cluster); addAdditionalHostedConnector(deployState, cluster); } } private void addAdditionalHostedConnector(DeployState deployState, ApplicationContainerCluster cluster) { - JettyHttpServer server = cluster.getHttp().getHttpServer(); + JettyHttpServer server = cluster.getHttp().getHttpServer().get(); String serverName = server.getComponentId().getName(); String proxyProtocol = deployState.getProperties().proxyProtocol(); @@ -356,39 +359,31 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return deployState.isHosted() && context.getApplicationType() == ApplicationType.DEFAULT && !isTesterApplication; } - private static void addHostedImplicitHttpIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) { + private static void addHostedImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) { if(cluster.getHttp() == null) { - Http http = deployState.getProperties().athenzDomain() - .map(tenantDomain -> createHostedImplicitHttpWithAccessControl(deployState, tenantDomain, cluster)) - .orElseGet(() -> createHostedImplicitHttpWithoutAccessControl(cluster)); - cluster.setHttp(http); + cluster.setHttp(new Http(new FilterChains(cluster))); } - if(cluster.getHttp().getHttpServer() == null) { + if(cluster.getHttp().getHttpServer().isEmpty()) { JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer")); cluster.getHttp().setHttpServer(defaultHttpServer); defaultHttpServer.addConnector(new ConnectorFactory("SearchServer", Defaults.getDefaults().vespaWebServicePort())); } } - private static Http createHostedImplicitHttpWithAccessControl( - DeployState deployState, AthenzDomain tenantDomain, ApplicationContainerCluster cluster) { + private void addHostedImplicitAccessControlIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) { + Http http = cluster.getHttp(); + if (http.getAccessControl().isPresent()) return; // access control added explicitly + AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null); + if (tenantDomain == null) return; // tenant domain not present, cannot add access control. this should eventually be a failure. AccessControl accessControl = new AccessControl.Builder(tenantDomain.value(), deployState.getDeployLogger()) .setHandlers(cluster) .readEnabled(false) .writeEnabled(false) .build(); - Http http = new Http(accessControl.getBindings(), accessControl); - FilterChains filterChains = new FilterChains(cluster); - filterChains.add(new Chain<>(FilterChains.emptyChainSpec(ACCESS_CONTROL_CHAIN_ID))); - http.setFilterChains(filterChains); - return http; - } - - private static Http createHostedImplicitHttpWithoutAccessControl(ApplicationContainerCluster cluster) { - Http http = new Http(Collections.emptyList()); - http.setFilterChains(new FilterChains(cluster)); - return http; + http.getFilterChains().add(new Chain<>(FilterChains.emptyChainSpec(ACCESS_CONTROL_CHAIN_ID))); + http.setAccessControl(accessControl); + http.getBindings().addAll(accessControl.getBindings()); } private Http buildHttp(DeployState deployState, ApplicationContainerCluster cluster, Element httpElement) { @@ -672,11 +667,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .vespaVersion(deployState.getWantedNodeVespaVersion()) .dockerImageRepo(deployState.getWantedDockerImageRepo()) .build(); - Capacity capacity = Capacity.fromCount(1, - Optional.empty(), - false, - ! deployState.getProperties().isBootstrap()); - HostResource host = hostSystem.allocateHosts(clusterSpec, capacity, 1, log).keySet().iterator().next(); + Capacity capacity = Capacity.from(new ClusterResources(1, 1, NodeResources.unspecified), + false, + ! deployState.getProperties().isBootstrap()); + HostResource host = hostSystem.allocateHosts(clusterSpec, capacity, log).keySet().iterator().next(); return singleHostContainerCluster(cluster, host, context); } } @@ -687,11 +681,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .dockerImageRepo(deployState.getWantedDockerImageRepo()) .build(); int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; - Capacity capacity = Capacity.fromCount(nodeCount, - Optional.empty(), - false, - !deployState.getProperties().isBootstrap()); - var hosts = hostSystem.allocateHosts(clusterSpec, capacity, 1, log); + Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified), + false, + !deployState.getProperties().isBootstrap()); + var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); return createNodesFromHosts(log, hosts, cluster); } return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); @@ -721,7 +714,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .build(); Map<HostResource, ClusterMembership> hosts = cluster.getRoot().hostSystem().allocateHosts(clusterSpec, - Capacity.fromRequiredNodeType(type), 1, log); + Capacity.fromRequiredNodeType(type), log); return createNodesFromHosts(context.getDeployLogger(), hosts, cluster); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 066fef727c5..6dd3e619ec2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -286,7 +286,7 @@ public class ContentCluster extends AbstractConfigProducer implements .orElse(NodesSpecification.nonDedicated(3, context)); Collection<HostResource> hosts = nodesSpecification.isDedicated() ? getControllerHosts(nodesSpecification, admin, clusterName, context) : - drawControllerHosts(nodesSpecification.count(), rootGroup, containers); + drawControllerHosts(nodesSpecification.minResources().nodes(), rootGroup, containers); clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone"), hosts, clusterName, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index f5a91297e9e..f93bf6fc872 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -241,7 +241,12 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ public Integer level = null; public void getConfig(ProtonConfig.Summary.Cache.Compression.Builder compression) { - if (type != null) compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name)); + if (type != null) compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name)); + if (level != null) compression.level(level); + } + + public void getConfig(ProtonConfig.Summary.Log.Compact.Compression.Builder compression) { + if (type != null) compression.type(ProtonConfig.Summary.Log.Compact.Compression.Type.Enum.valueOf(type.name)); if (level != null) compression.level(level); } @@ -281,6 +286,12 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ } } + public void getConfig(ProtonConfig.Summary.Log.Compact.Builder compact) { + if (compression != null) { + compression.getConfig(compact.compression); + } + } + public void getConfig(ProtonConfig.Summary.Log.Chunk.Builder chunk) { if (outputInt) { if (maxSize!=null) chunk.maxbytes(maxSize.intValue()); @@ -288,7 +299,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ throw new IllegalStateException("Fix this, chunk does not have long types"); } if (compression != null) { - compression.getConfig(chunk.compression); + compression.getConfig(chunk.compression); } } } @@ -303,6 +314,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ if (minFileSizeFactor!=null) log.minfilesizefactor(minFileSizeFactor); if (chunk != null) { chunk.getConfig(log.chunk); + chunk.getConfig(log.compact); } } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index cca56c209c8..3560cf2cd84 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -1816,6 +1816,7 @@ Object indexBody(IndexOperation index) : | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); } | <ENABLE_BM25> { index.setEnableBm25(true); } + | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { index.setDistanceMetric(str); } | hnswIndex(index) { } ) { return null; } @@ -1841,7 +1842,6 @@ void hnswIndexBody(HnswIndexParams.Builder params) : } { ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); } - | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { params.setDistanceMetric(str); } | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); } ) } diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index c47983adc12..878faabfec1 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -17,14 +17,14 @@ anyElement = element * { JavaId = xsd:string { pattern = "([a-zA-Z_$][a-zA-Z\d_$]*\.)*[a-zA-Z_$][a-zA-Z\d_$]*" } Nodes = element nodes { - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute docker-image { xsd:string }? & Resources? } Resources = element resources { - attribute vcpu { xsd:double { minExclusive = "0.0" } } & + attribute vcpu { xsd:double { minExclusive = "0.0" } | xsd:string } & attribute memory { xsd:string } & attribute disk { xsd:string } & attribute disk-speed { xsd:string }? & @@ -32,7 +32,7 @@ Resources = element resources { } OptionalDedicatedNodes = element nodes { - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute docker-image { xsd:string }? & diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 726fa849c00..3c8b60fb84b 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -239,7 +239,7 @@ NodesOfContainerCluster = element nodes { attribute type { xsd:string } | ( - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute exclusive { xsd:boolean }? & diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index ee451185415..b1821680b14 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -221,11 +221,11 @@ ContentNodes = element nodes { attribute vespamalloc-debug-stacktrace { xsd:string }? & ( ( - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute docker-image { xsd:string }? & - attribute groups { xsd:positiveInteger }? + attribute groups { xsd:positiveInteger | xsd:string }? ) | ContentNode + @@ -266,12 +266,12 @@ Group = element group { | ( element nodes { - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute exclusive { xsd:boolean }? & attribute docker-image { xsd:string }? & - attribute groups { xsd:positiveInteger }? + attribute groups { xsd:positiveInteger | xsd:string }? } ) | diff --git a/config-model/src/test/derived/hnsw_index/test.sd b/config-model/src/test/derived/hnsw_index/test.sd index 3b954e74fc5..207ed764a87 100644 --- a/config-model/src/test/derived/hnsw_index/test.sd +++ b/config-model/src/test/derived/hnsw_index/test.sd @@ -3,9 +3,9 @@ search test { field t1 type tensor(x[128]) { indexing: attribute | index index { + distance-metric: angular hnsw { max-links-per-node: 32 - distance-metric: angular neighbors-to-explore-at-insert: 300 } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 1670ac23ba4..7208d8c5fc1 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1205,6 +1205,62 @@ public class ModelProvisioningTest { } @Test + public void testRequestingRangesMin() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='container'>" + + " <nodes count='[4, 6]'>" + + " <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" + + " </nodes>" + + " </container>" + + " <content version='1.0' id='foo'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='[6, 20]' groups='[3,4]'>" + + " <resources vcpu='8' memory='200Gb' disk='1Pb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + + int totalHosts = 10; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(new NodeResources(11.5, 10, 30, 0.3), 6); + tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20); + VespaModel model = tester.createModel(services, true); + assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); + } + + @Test + public void testRequestingRangesMax() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='container'>" + + " <nodes count='[4, 6]'>" + + " <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" + + " </nodes>" + + " </container>" + + " <content version='1.0' id='foo'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='[6, 20]' groups='[3,4]'>" + + " <resources vcpu='8' memory='200Gb' disk='1Pb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + + int totalHosts = 26; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 6); + tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20); + VespaModel model = tester.createModel(services, true, true); + assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); + } + + @Test public void testContainerOnly() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + @@ -1308,10 +1364,11 @@ public class ModelProvisioningTest { " </http>" + "</container>"; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(1); + tester.addHosts(2); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getHosts().size()); + assertEquals(2, model.getHosts().size()); assertEquals(1, model.getContainerClusters().size()); + assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); } @Test @@ -1374,7 +1431,7 @@ public class ModelProvisioningTest { } @Test - public void testNoNodeTagMeans1Node() { + public void testNoNodeTagMeansTwoNodes() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + @@ -1389,31 +1446,6 @@ public class ModelProvisioningTest { " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(1); - VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().hostSystem().getHosts().size()); - assertEquals(1, model.getAdmin().getSlobroks().size()); - assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); - assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); - } - - @Test - public void testNoNodeTagMeansTwoNodesInContainerClusterWithFeatureFlag() { - String services = - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services>" + - " <container id='foo' version='1.0'>" + - " <search/>" + - " <document-api/>" + - " </container>" + - " <content version='1.0' id='bar'>" + - " <documents>" + - " <document type='type1' mode='index'/>" + - " </documents>" + - " </content>" + - "</services>"; - VespaModelTester tester = new VespaModelTester(); - tester.setUseDedicatedNodesWhenUnspecified(true); tester.addHosts(3); VespaModel model = tester.createModel(services, true); assertEquals(3, model.getRoot().hostSystem().getHosts().size()); @@ -1423,7 +1455,7 @@ public class ModelProvisioningTest { } @Test - public void testNoNodeTagMeans1NodeNoContent() { + public void testNoNodeTagMeansTwoNodesNoContent() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + @@ -1433,11 +1465,11 @@ public class ModelProvisioningTest { " </container>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(1); + tester.addHosts(2); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().hostSystem().getHosts().size()); - assertEquals(1, model.getAdmin().getSlobroks().size()); - assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); + assertEquals(2, model.getRoot().hostSystem().getHosts().size()); + assertEquals(2, model.getAdmin().getSlobroks().size()); + assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java index ead4e586d9f..9f57b22fd58 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java @@ -31,6 +31,9 @@ public class NearestNeighborTestCase extends AbstractExportingTestCase { } catch (QueryException e) { // success assertEquals("Invalid request parameter", e.getMessage()); + } catch (RuntimeException e) { + e.printStackTrace(); + throw e; } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java index d687590faf2..e3dcc925e5e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java @@ -2,13 +2,13 @@ package com.yahoo.searchdefinition.document; +import java.util.Optional; import org.junit.Test; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; -import static com.yahoo.searchdefinition.document.HnswIndexParams.DistanceMetric; public class HnswIndexParamsTestCase { @@ -18,35 +18,27 @@ public class HnswIndexParamsTestCase { var builder = new HnswIndexParams.Builder(); builder.setMaxLinksPerNode(7); var one = builder.build(); - builder.setDistanceMetric("angular"); - var two = builder.build(); builder.setNeighborsToExploreAtInsert(42); var three = builder.build(); builder.setMaxLinksPerNode(17); - builder.setDistanceMetric("geodegrees"); builder.setNeighborsToExploreAtInsert(500); var four = builder.build(); assertThat(empty.maxLinksPerNode(), is(16)); - assertThat(empty.distanceMetric(), is(DistanceMetric.EUCLIDEAN)); assertThat(empty.neighborsToExploreAtInsert(), is(200)); assertThat(one.maxLinksPerNode(), is(7)); - assertThat(two.distanceMetric(), is(DistanceMetric.ANGULAR)); assertThat(three.neighborsToExploreAtInsert(), is(42)); assertThat(four.maxLinksPerNode(), is(17)); - assertThat(four.distanceMetric(), is(DistanceMetric.GEODEGREES)); assertThat(four.neighborsToExploreAtInsert(), is(500)); - var five = four.overrideFrom(empty); + var five = four.overrideFrom(Optional.of(empty)); assertThat(five.maxLinksPerNode(), is(17)); - assertThat(five.distanceMetric(), is(DistanceMetric.GEODEGREES)); assertThat(five.neighborsToExploreAtInsert(), is(500)); - var six = four.overrideFrom(two); + var six = four.overrideFrom(Optional.of(one)); assertThat(six.maxLinksPerNode(), is(7)); - assertThat(six.distanceMetric(), is(DistanceMetric.ANGULAR)); assertThat(six.neighborsToExploreAtInsert(), is(500)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java index 4b7f727ff63..cf1ae637cf9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -109,7 +109,13 @@ public class VespaModelFactoryTest { } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + return prepare(cluster, capacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { return List.of(new HostSpec(hostName, List.of(), ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java index 060fff5bf66..73dd1ca3f3b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.collections.CollectionUtil; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; -import com.yahoo.vespa.model.content.DispatchTuning; import com.yahoo.vespa.model.search.Tuning; import org.junit.Test; import org.w3c.dom.Element; @@ -228,6 +227,8 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest { assertEquals(cfg.summary().log().chunk().maxbytes(), 256); assertEquals(cfg.summary().log().chunk().compression().type(), ProtonConfig.Summary.Log.Chunk.Compression.Type.LZ4); assertEquals(cfg.summary().log().chunk().compression().level(), 5); + assertEquals(cfg.summary().log().compact().compression().type(), ProtonConfig.Summary.Log.Compact.Compression.Type.LZ4); + assertEquals(cfg.summary().log().compact().compression().level(), 5); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index b1e63628852..28e23ce3222 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -3,8 +3,13 @@ package com.yahoo.vespa.model.container.xml; import com.google.common.collect.ImmutableSet; import com.yahoo.collections.CollectionUtil; +import com.yahoo.component.ComponentId; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.container.jdisc.state.StateHandler; +import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.http.AccessControl; import com.yahoo.vespa.model.container.http.Http; @@ -16,14 +21,19 @@ import org.w3c.dom.Element; import java.util.Collection; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -229,6 +239,58 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { } + + @Test + public void access_control_is_implicitly_added_for_hosted_apps() { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + nodesXml, + "</container>" ); + AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); + DeployState state = new DeployState.Builder().properties( + new TestProperties() + .setAthenzDomain(tenantDomain) + .setHostedVespa(true)) + .build(); + createModel(root, state, null, clusterElem); + Optional<AccessControl> maybeAccessControl = + ((ApplicationContainer) root.getProducer("container/container.0")).getHttp().getAccessControl(); + assertThat(maybeAccessControl.isPresent(), is(true)); + AccessControl accessControl = maybeAccessControl.get(); + assertThat(accessControl.writeEnabled, is(false)); + assertThat(accessControl.readEnabled, is(false)); + assertThat(accessControl.domain, equalTo(tenantDomain.value())); + } + + @Test + public void access_control_is_implicitly_added_for_hosted_apps_with_existing_http_element() { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + " <http>", + " <server port='" + getDefaults().vespaWebServicePort() + "' id='main' />", + " <filtering>", + " <filter id='outer' />", + " <request-chain id='myChain'>", + " <filter id='inner' />", + " </request-chain>", + " </filtering>", + " </http>", + nodesXml, + "</container>" ); + AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); + DeployState state = new DeployState.Builder().properties( + new TestProperties() + .setAthenzDomain(tenantDomain) + .setHostedVespa(true)) + .build(); + createModel(root, state, null, clusterElem); + Http http = ((ApplicationContainer) root.getProducer("container/container.0")).getHttp(); + assertThat(http.getAccessControl().isPresent(), is(true)); + assertThat(http.getFilterChains().hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID), is(true)); + assertThat(http.getFilterChains().hasChain(ComponentId.fromString("myChain")), is(true)); + } + + private String httpWithExcludedBinding(String excludedBinding) { return joinLines( " <http>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 73db6e35428..dcd1c46e52f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -14,7 +14,6 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; -import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.RegionName; @@ -44,7 +43,6 @@ import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.http.AccessControl; import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; @@ -626,7 +624,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .setMultitenant(true) .setHostedVespa(true)) .build()); - assertEquals(1, model.hostSystem().getHosts().size()); + assertEquals(2, model.hostSystem().getHosts().size()); } @Test(expected = IllegalArgumentException.class) @@ -811,7 +809,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0"); // Verify that there are two connectors - List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().getConnectorFactories(); + List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories(); assertEquals(2, connectorFactories.size()); List<Integer> ports = connectorFactories.stream() .map(ConnectorFactory::getListenPort) @@ -835,28 +833,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertThat(connectorConfig.ssl().caCertificate(), isEmptyString()); } - @Test - public void access_control_is_implicitly_added_for_hosted_apps() { - Element clusterElem = DomBuilderTest.parse( - "<container version='1.0'>", - nodesXml, - "</container>" ); - AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); - DeployState state = new DeployState.Builder().properties( - new TestProperties() - .setAthenzDomain(tenantDomain) - .setHostedVespa(true)) - .build(); - createModel(root, state, null, clusterElem); - Optional<AccessControl> maybeAccessControl = - ((ApplicationContainer) root.getProducer("container/container.0")).getHttp().getAccessControl(); - assertThat(maybeAccessControl.isPresent(), is(true)); - AccessControl accessControl = maybeAccessControl.get(); - assertThat(accessControl.writeEnabled, is(false)); - assertThat(accessControl.readEnabled, is(false)); - assertThat(accessControl.domain, equalTo(tenantDomain.value())); - } - private Element generateContainerElementWithRenderer(String rendererId) { return DomBuilderTest.parse( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index fd837c6dea3..cdfd9fab194 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -48,7 +48,6 @@ public class VespaModelTester { private Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>(); private ApplicationId applicationId = ApplicationId.defaultId(); private boolean useDedicatedNodeForLogserver = false; - private boolean useDedicatedNodesWhenUnspecified = false; public VespaModelTester() { this(new NullConfigModelRegistry()); @@ -98,10 +97,6 @@ public class VespaModelTester { this.useDedicatedNodeForLogserver = useDedicatedNodeForLogserver; } - public void setUseDedicatedNodesWhenUnspecified(boolean useDedicatedNodesWhenUnspecified) { - this.useDedicatedNodesWhenUnspecified = useDedicatedNodesWhenUnspecified; - } - /** Creates a model which uses 0 as start index and fails on out of capacity */ public VespaModel createModel(String services, String ... retiredHostNames) { return createModel(Zone.defaultZone(), services, true, retiredHostNames); @@ -109,41 +104,48 @@ public class VespaModelTester { /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, 0, retiredHostNames); + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0, retiredHostNames); + } + + /** Creates a model which uses 0 as start index */ + public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, String ... retiredHostNames) { + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, 0, retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, startIndexForClusters, retiredHostNames); + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, startIndexForClusters, retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(zone, services, failOnOutOfCapacity, 0, retiredHostNames); + return createModel(zone, services, failOnOutOfCapacity, false, 0, retiredHostNames); } /** * Creates a model using the hosts already added to this * * @param services the services xml string + * @param useMaxResources false to use the minmal resources (when given a range), true to use max * @param failOnOutOfCapacity whether we should get an exception when not enough hosts of the requested flavor * is available or if we should just silently receive a smaller allocation * @return the resulting model */ - public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { + public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, boolean useMaxResources, + int startIndexForClusters, String ... retiredHostNames) { VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; - HostProvisioner provisioner = hosted ? - new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) : + HostProvisioner provisioner = hosted ? + new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, useMaxResources, + startIndexForClusters, retiredHostNames) : new SingleNodeProvisioner(); TestProperties properties = new TestProperties() .setMultitenant(true) .setHostedVespa(hosted) .setApplicationId(applicationId) - .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver) - .setUseDedicatedNodesWhenUnspecified(useDedicatedNodesWhenUnspecified); + .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver); DeployState deployState = new DeployState.Builder() .applicationPackage(appPkg) diff --git a/config-model/src/test/schema-test-files/services-hosted.xml b/config-model/src/test/schema-test-files/services-hosted.xml index 07839239c81..71a07926240 100644 --- a/config-model/src/test/schema-test-files/services-hosted.xml +++ b/config-model/src/test/schema-test-files/services-hosted.xml @@ -7,7 +7,7 @@ </admin> <container id="container1" version="1.0"> - <nodes count="5" required="true"> + <nodes count="[5,7]" required="true"> <resources vcpu="1.2" memory="10Gb" disk="0.3 TB"/> </nodes> </container> @@ -27,4 +27,11 @@ </nodes> </content> + <content id="ml" version="1.0"> + <redundancy>2</redundancy> + <nodes count="[10,20]" flavor="large" groups="[1,3]"> + <resources vcpu="[3.0, 4]" memory="[32000.0Mb, 33Gb]" disk="[300 Gb, 1Tb]"/> + </nodes> + </content> + </services> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 59d6ec8feb8..48b4e9d91bc 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -11,26 +11,37 @@ import java.util.Optional; */ public final class Capacity { - private final int nodeCount; + /** Resources should stay between these values, inclusive */ + private final ClusterResources min, max; private final boolean required; private final boolean canFail; - private final Optional<NodeResources> nodeResources; - private final NodeType type; - private Capacity(int nodeCount, Optional<NodeResources> nodeResources, boolean required, boolean canFail, NodeType type) { - this.nodeCount = nodeCount; + private Capacity(ClusterResources min, ClusterResources max, boolean required, boolean canFail, NodeType type) { + if (max.smallerThan(min)) + throw new IllegalArgumentException("The max capacity must be larger than the min capacity, but got min " + + min + " and max " + max); + this.min = min; + this.max = max; this.required = required; this.canFail = canFail; - this.nodeResources = nodeResources; this.type = type; } /** Returns the number of nodes requested */ - public int nodeCount() { return nodeCount; } + @Deprecated // TODO: Remove after April 2020 + public int nodeCount() { return min.nodes(); } + + /** Returns the number of nodes requested (across all groups), or 0 if not specified */ + @Deprecated // TODO: Remove after April 2020 + public int nodes() { return min.nodes(); } + + /** Returns the number of groups requested, or 0 if not specified */ + @Deprecated // TODO: Remove after April 2020 + public int groups() { return min.groups(); } /** * The node flavor requested, or empty if no legacy flavor name has been used. @@ -38,14 +49,21 @@ public final class Capacity { * * @deprecated use nodeResources instead */ - @Deprecated + @Deprecated // TODO: Remove after March 2020 public Optional<String> flavor() { if (nodeResources().isEmpty()) return Optional.empty(); - return nodeResources.map(n -> n.toString()); + return Optional.of(min.nodeResources().toString()); } /** Returns the resources requested for each node, or empty to leave this decision to provisioning */ - public Optional<NodeResources> nodeResources() { return nodeResources; } + @Deprecated // TODO: Remove after March 2020 + public Optional<NodeResources> nodeResources() { + if (min.nodeResources() == NodeResources.unspecified) return Optional.empty(); + return Optional.of(min.nodeResources()); + } + + public ClusterResources minResources() { return min; } + public ClusterResources maxResources() { return max; } /** Returns whether the requested number of nodes must be met exactly for a request for this to succeed */ public boolean isRequired() { return required; } @@ -64,32 +82,53 @@ public final class Capacity { */ public NodeType type() { return type; } + public Capacity withGroups(int groups) { + return new Capacity(min.withGroups(groups), max.withGroups(groups), required, canFail, type); + } + @Override public String toString() { - return nodeCount + " nodes " + (nodeResources.isPresent() ? nodeResources.get() : "with default resources" ); + return (required ? "required " : "") + + (min.equals(max) ? min : "between " + min + " and " + max); } - /** Creates this from a desired node count: The request may be satisfied with a smaller number of nodes. */ - public static Capacity fromNodeCount(int capacity) { - return fromCount(capacity, Optional.empty(), false, true); + /** Create a non-required, failable capacity request */ + public static Capacity from(ClusterResources resources) { + return from(resources, false, true); + } + + public static Capacity from(ClusterResources resources, boolean required, boolean canFail) { + return from(resources, required, canFail, NodeType.tenant); + } + + public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail) { + return new Capacity(min, max, required, canFail, NodeType.tenant); } /** Create a non-required, failable capacity request */ - public static Capacity fromCount(int nodeCount, NodeResources resources) { - return fromCount(nodeCount, resources, false, true); + @Deprecated // TODO: Remove after April 2020 + public static Capacity fromCount(int nodes, NodeResources resources) { + return fromCount(nodes, resources, false, true); } - public static Capacity fromCount(int nodeCount, NodeResources resources, boolean required, boolean canFail) { - return new Capacity(nodeCount, Optional.of(resources), required, canFail, NodeType.tenant); + @Deprecated // TODO: Remove after April 2020 + public static Capacity fromCount(int nodes, NodeResources resources, boolean required, boolean canFail) { + return fromCount(nodes, Optional.of(resources), required, canFail); } - public static Capacity fromCount(int nodeCount, Optional<NodeResources> resources, boolean required, boolean canFail) { - return new Capacity(nodeCount, resources, required, canFail, NodeType.tenant); + @Deprecated // TODO: Remove after April 2020 + public static Capacity fromCount(int nodes, Optional<NodeResources> resources, boolean required, boolean canFail) { + return from(new ClusterResources(nodes, 0, resources.orElse(NodeResources.unspecified)), + required, canFail, NodeType.tenant); } /** Creates this from a node type */ public static Capacity fromRequiredNodeType(NodeType type) { - return new Capacity(0, Optional.empty(), true, false, type); + return from(new ClusterResources(0, 0, NodeResources.unspecified), true, false, type); + } + + private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) { + return new Capacity(resources, resources, required, canFail, type); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java index b777a13af7c..178bbac9e64 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java @@ -110,8 +110,6 @@ public class ClusterMembership { @Override public String toString() { return stringValue(); } - // TODO: Remove when when 7.195 is oldest model version in use - @Deprecated public static ClusterMembership from(String stringValue, Version vespaVersion) { return new ClusterMembership(stringValue, vespaVersion, Optional.empty()); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java new file mode 100644 index 00000000000..a4ed22c5266 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java @@ -0,0 +1,78 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import java.util.Objects; + +/** + * The resources of a cluster + * + * @author bratseth + */ +public class ClusterResources { + + /** The node count in the cluster */ + private final int nodes; + + /** The number of node groups in the cluster */ + private final int groups; + + /** The resources of each node in the cluster */ + private final NodeResources nodeResources; + + public ClusterResources(int nodes, int groups, NodeResources nodeResources) { + if (nodes > 0 && groups > 0 && nodes % groups != 0) + throw new IllegalArgumentException("The number of nodes (" + nodes + + ") must be divisible by the number of groups (" + groups + ")"); + this.nodes = nodes; + this.groups = groups; + this.nodeResources = Objects.requireNonNull(nodeResources); + } + + /** Returns the total number of allocated nodes (over all groups) */ + public int nodes() { return nodes; } + public int groups() { return groups; } + public NodeResources nodeResources() { return nodeResources; } + + public ClusterResources with(NodeResources resources) { return new ClusterResources(nodes, groups, resources); } + public ClusterResources withGroups(int groups) { return new ClusterResources(nodes, groups, nodeResources); } + + /** Returns true if this is smaller than the given resources in any dimension */ + public boolean smallerThan(ClusterResources other) { + if (this.nodes < other.nodes) return true; + if (this.groups < other.groups) return true; + if ( ! this.nodeResources.justNumbers().satisfies(other.nodeResources.justNumbers())) return true; + return false; + } + + /** Returns true if this is within the given limits (inclusive) */ + public boolean isWithin(ClusterResources min, ClusterResources max) { + if (this.smallerThan(min)) return false; + if (max.smallerThan(this)) return false; + return true; + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof ClusterResources)) return false; + + ClusterResources other = (ClusterResources)o; + if (other.nodes != this.nodes) return false; + if (other.groups != this.groups) return false; + if ( ! other.nodeResources.equals(this.nodeResources)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(nodes, groups, nodeResources); + } + + @Override + public String toString() { + return nodes + " nodes" + + (groups > 1 ? " (in " + groups + " groups)" : "") + + " with " + nodeResources; + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 66a2ff411fe..147066b82e7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -77,27 +77,13 @@ public final class ClusterSpec { return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo); } - // TODO: Remove when when 7.195 is oldest model version in use - // TODO: Add @Deprecated when internal repo has been updated to not use this method - // @Deprecated - public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive, - Optional<Id> combinedId) { - return request(type, id, vespaVersion, exclusive, combinedId, Optional.empty()); - } - + // TODO: Remove when when 7.200 is oldest model version in use public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive, Optional<Id> combinedId, Optional<String> dockerImageRepo) { return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive, combinedId, dockerImageRepo); } - // TODO: Remove when when 7.195 is oldest model version in use - // TODO: Add @Deprecated when internal repo has been updated to not use this method - // @Deprecated - public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive, - Optional<Id> combinedId) { - return from(type, id, groupId, vespaVersion, exclusive, combinedId, Optional.empty()); - } - + // TODO: Remove when when 7.200 is oldest model version in use public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive, Optional<Id> combinedId, Optional<String> dockerImageRepo) { return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive, combinedId, dockerImageRepo); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 6399352a6ec..5fc05a87a7d 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -10,6 +10,8 @@ import java.util.Objects; */ public class NodeResources { + public static final NodeResources unspecified = new NodeResources(0, 0, 0, 0); + public enum DiskSpeed { fast, // Has/requires SSD disk or similar speed @@ -112,26 +114,32 @@ public class NodeResources { public StorageType storageType() { return storageType; } public NodeResources withVcpu(double vcpu) { + if (vcpu == this.vcpu) return this; return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withMemoryGb(double memoryGb) { + if (memoryGb == this.memoryGb) return this; return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withDiskGb(double diskGb) { + if (diskGb == this.diskGb) return this; return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withBandwidthGbps(double bandwidthGbps) { + if (bandwidthGbps == this.bandwidthGbps) return this; return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } - public NodeResources with(DiskSpeed speed) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, speed, storageType); + public NodeResources with(DiskSpeed diskSpeed) { + if (diskSpeed == this.diskSpeed) return this; + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources with(StorageType storageType) { + if (storageType == this.storageType) return this; return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java index 6be1d49ebd3..e308d631442 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java @@ -19,10 +19,12 @@ public interface Provisioner { * @param applicationId the application requesting hosts * @param cluster the specification of the cluster to allocate nodes for * @param capacity the capacity requested - * @param groups the number of node groups to divide the requested capacity into * @param logger a logger which receives messages which are returned to the requestor * @return the specification of the hosts allocated */ + List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger); + + @Deprecated // TODO: Remove after April 2020 List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger); /** @@ -40,7 +42,6 @@ public interface Provisioner { * @param transaction Transaction with operations to commit together with any operations done within the provisioner. * @param application the application to remove */ - @SuppressWarnings("deprecation") void remove(NestedTransaction transaction, ApplicationId application); /** diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java new file mode 100644 index 00000000000..326ed7317f6 --- /dev/null +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java @@ -0,0 +1,48 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class CapacityTest { + + @Test + public void testCapacityValidation() { + // Equal min and max is allowed + Capacity.from(new ClusterResources(4, 2, new NodeResources(1,2,3,4)), + new ClusterResources(4, 2, new NodeResources(1,2,3,4)), + false, true); + assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1,2,3,4)), + new ClusterResources(2, 2, new NodeResources(1,2,3,4))); + assertValidationFailure(new ClusterResources(4, 4, new NodeResources(1,2,3,4)), + new ClusterResources(4, 2, new NodeResources(1,2,3,4))); + assertValidationFailure(new ClusterResources(4, 2, new NodeResources(2,2,3,4)), + new ClusterResources(4, 2, new NodeResources(1,2,3,4))); + assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1,3,3,4)), + new ClusterResources(4, 2, new NodeResources(1,2,3,4))); + assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1,2,4,4)), + new ClusterResources(4, 2, new NodeResources(1,2,3,4))); + assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1,2,3,5)), + new ClusterResources(4, 2, new NodeResources(1,2,3,4))); + // It's enough than one dimension is smaller also when the others are larger + assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1,2,3,4)), + new ClusterResources(8, 4, new NodeResources(2,1,6,8))); + } + + private void assertValidationFailure(ClusterResources min, ClusterResources max) { + try { + Capacity.from(min, max, false, true); + fail("Expected exception with min " + min + " and max " + max); + } + catch (IllegalArgumentException e) { + assertEquals("The max capacity must be larger than the min capacity, but got min " + min + " and max " + max, + e.getMessage()); + } + } + +} diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index 33c568061fe..cc496a99c20 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -4,6 +4,9 @@ namespace=cloud.config +# Enable proxy-protocol for nginx upstreams +nginxUpstreamProxyProtocol bool default=false + # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index b10726cd73b..95c8733b540 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -314,10 +314,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye tenant.getLocalSessionRepo().addSession(newSession); return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock, - false /* don't validate as this is already deployed */, - newSession.getDockerImageRepository(), - newSession.getVespaVersion(), - bootstrap)); + false /* don't validate as this is already deployed */, bootstrap)); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 439408fc01c..63b1df3d634 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -62,7 +62,6 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final StateMonitor stateMonitor; private final VipStatus vipStatus; private final ConfigserverConfig configserverConfig; - private final SuperModelManager superModelManager; private final Duration maxDurationOfRedeployment; private final Duration sleepTimeWhenRedeployingFails; private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails; @@ -71,32 +70,29 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable @SuppressWarnings("unused") @Inject public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, - VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, - SuperModelManager superModelManager) { + VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus) { this(applicationRepository, server, versionState, stateMonitor, vipStatus, BOOTSTRAP_IN_CONSTRUCTOR, EXIT_JVM, applicationRepository.configserverConfig().hostedVespa() ? VipStatusMode.VIP_STATUS_FILE - : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY, - superModelManager); + : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY); } // For testing only ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, - StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, VipStatusMode vipStatusMode) { - this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode, null); + StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, VipStatusMode vipStatusMode) { + this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode); } private ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails, - VipStatusMode vipStatusMode, SuperModelManager superModelManager) { + VipStatusMode vipStatusMode) { this.applicationRepository = applicationRepository; this.server = server; this.versionState = versionState; this.stateMonitor = stateMonitor; this.vipStatus = vipStatus; this.configserverConfig = applicationRepository.configserverConfig(); - this.superModelManager = superModelManager; this.maxDurationOfRedeployment = Duration.ofSeconds(configserverConfig.maxDurationOfBootstrap()); this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails()); this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 9a46ce099f8..b3c5b38b9ba 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; import com.yahoo.log.LogLevel; @@ -56,6 +57,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { /** True if this deployment is done to bootstrap the config server */ private final boolean isBootstrap; + /** The (optional) Athenz domain this application should use */ + private final Optional<AthenzDomain> athenzDomain; + private boolean prepared = false; /** Whether this model should be validated (only takes effect if prepared=false) */ @@ -64,9 +68,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment { private boolean ignoreSessionStaleFailure = false; private Deployment(LocalSession session, ApplicationRepository applicationRepository, - Optional<Provisioner> hostProvisioner, Tenant tenant, - Duration timeout, Clock clock, boolean prepared, boolean validate, - Optional<String> dockerImageRepository, Version version, boolean isBootstrap) { + Optional<Provisioner> hostProvisioner, Tenant tenant, Duration timeout, + Clock clock, boolean prepared, boolean validate, boolean isBootstrap) { this.session = session; this.applicationRepository = applicationRepository; this.hostProvisioner = hostProvisioner; @@ -75,26 +78,24 @@ public class Deployment implements com.yahoo.config.provision.Deployment { this.clock = clock; this.prepared = prepared; this.validate = validate; - this.dockerImageRepository = dockerImageRepository; - this.version = version; + this.dockerImageRepository = session.getDockerImageRepository(); + this.version = session.getVespaVersion(); this.isBootstrap = isBootstrap; + this.athenzDomain = session.getAthenzDomain(); } public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, Optional<Provisioner> hostProvisioner, Tenant tenant, - Duration timeout, Clock clock, boolean validate, - Optional<String> dockerImageRepository, Version version, - boolean isBootstrap) { - return new Deployment(session, applicationRepository, hostProvisioner, tenant, - timeout, clock, false, validate, dockerImageRepository, version, isBootstrap); + Duration timeout, Clock clock, boolean validate, boolean isBootstrap) { + return new Deployment(session, applicationRepository, hostProvisioner, tenant, timeout, clock, false, + validate, isBootstrap); } public static Deployment prepared(LocalSession session, ApplicationRepository applicationRepository, Optional<Provisioner> hostProvisioner, Tenant tenant, Duration timeout, Clock clock, boolean isBootstrap) { return new Deployment(session, applicationRepository, hostProvisioner, tenant, - timeout, clock, true, true, session.getDockerImageRepository(), - session.getVespaVersion(), isBootstrap); + timeout, clock, true, true, isBootstrap); } public void setIgnoreSessionStaleFailure(boolean ignoreSessionStaleFailure) { @@ -114,6 +115,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { .vespaVersion(version.toString()) .isBootstrap(isBootstrap); dockerImageRepository.ifPresent(params::dockerImageRepository); + athenzDomain.ifPresent(params::athenzDomain); session.prepare(logger, params.build(), Optional.empty(), tenant.getPath(), clock.instant()); this.prepared = true; } @@ -147,7 +149,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() + " activated successfully using " + - (hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner") + + (hostProvisioner.isPresent() ? hostProvisioner.get().getClass().getSimpleName() : "no host provisioner") + ". Config generation " + session.getMetaData().getGeneration() + ". File references used: " + applicationRepository.getFileReferences(applicationId)); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java index 197b90e322c..c971fdd7b13 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java @@ -102,7 +102,7 @@ public class ClusterMetricsRetriever { return slime; } catch (IOException e) { // Usually caused by applications being deleted during metric retrieval - log.warning("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e)); + log.info("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e)); return new Slime(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index 6366576e163..4fe2ec129d3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -9,7 +9,9 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import java.util.Collections; import java.util.Comparator; @@ -32,14 +34,17 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; + private final BooleanFlag nginxUpstreamProxyProtocol; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; + this.nginxUpstreamProxyProtocol = Flags.NGINX_UPSTREAM_PROXY_PROTOCOL.bindTo(flagSource); } @Override public void getConfig(LbServicesConfig.Builder builder) { + builder.nginxUpstreamProxyProtocol(nginxUpstreamProxyProtocol.value()); models.keySet().stream() .sorted() .forEach(tenant -> { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java index ee4cc4a3043..daacdf9fd50 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -36,8 +36,14 @@ public class ProvisionerAdapter implements HostProvisioner { } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - return provisioner.prepare(applicationId, cluster, capacity, groups, logger); + return provisioner.prepare(applicationId, cluster, capacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + return provisioner.prepare(applicationId, cluster, capacity, logger); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java index 26d322665f0..8ddfb1e8b09 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java @@ -33,8 +33,15 @@ public class StaticProvisioner implements HostProvisioner { throw new UnsupportedOperationException("Allocating a single host from provisioning info is not supported"); } + @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + return prepare(cluster, capacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { List<HostSpec> hostsAlreadyAllocatedToCluster = allocatedHosts.getHosts().stream() .filter(host -> host.membership().isPresent() && matches(host.membership().get().cluster(), cluster)) @@ -42,7 +49,7 @@ public class StaticProvisioner implements HostProvisioner { if ( ! hostsAlreadyAllocatedToCluster.isEmpty()) return hostsAlreadyAllocatedToCluster; else - return fallback.prepare(cluster, capacity, groups, logger); + return fallback.prepare(cluster, capacity, logger); } private boolean matches(ClusterSpec nodeCluster, ClusterSpec requestedCluster) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index e9d1f39f788..825ae0d8d92 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.transaction.AbstractTransaction; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -142,6 +143,10 @@ public class LocalSession extends Session implements Comparable<LocalSession> { zooKeeperClient.writeDockerImageRepository(dockerImageRepository); } + public void setAthenzDomain(Optional<AthenzDomain> athenzDomain) { + zooKeeperClient.writeAthenzDomain(athenzDomain); + } + public enum Mode { READ, WRITE } @@ -156,6 +161,8 @@ public class LocalSession extends Session implements Comparable<LocalSession> { public Version getVespaVersion() { return zooKeeperClient.readVespaVersion(); } + public Optional<AthenzDomain> getAthenzDomain() { return zooKeeperClient.readAthenzDomain(); } + public AllocatedHosts getAllocatedHosts() { return zooKeeperClient.getAllocatedHosts(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index 6a671648b27..f7ed801ddbd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -163,6 +163,11 @@ public final class PrepareParams { return this; } + public Builder athenzDomain(AthenzDomain athenzDomain) { + this.athenzDomain = Optional.of(athenzDomain); + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index 87dead8eed1..06a3dfa8777 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -129,9 +129,11 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { long activeSessionId = getActiveSessionId(existingApplicationId); logger.log(LogLevel.DEBUG, "Create from existing application id " + existingApplicationId + ", active session id is " + activeSessionId); LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); + // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() session.setApplicationId(existingApplicationId); session.setVespaVersion(existingSession.getVespaVersion()); session.setDockerImageRepository(existingSession.getDockerImageRepository()); + session.setAthenzDomain(existingSession.getAthenzDomain()); return session; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index b88fdc90316..6c77964a58b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -300,6 +300,7 @@ public class SessionPreparer { ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger); try { zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); + // Note: When changing the below you need to also change similar calls in SessionFactoryImpl.createSessionFromExisting() zooKeeperClient.writeApplicationId(applicationId); zooKeeperClient.writeVespaVersion(vespaVersion); zooKeeperClient.writeDockerImageRepository(dockerImageRepository); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 2a1254d0d8d..654d811a31f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.component.Version; import com.yahoo.config.provision.Zone; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.handler.ClustersStatus; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.config.HealthMonitorConfig; @@ -235,6 +236,7 @@ public class ConfigServerBootstrapTest { private VipStatus createVipStatus(StateMonitor stateMonitor) { return new VipStatus(new QrSearchersConfig.Builder().build(), + new VipStatusConfig.Builder().build(), new ClustersStatus(), stateMonitor); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 85f07da0325..84987bce32e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -221,26 +221,26 @@ public class DeployTester { * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ public PrepareResult deployApp(String applicationPath, String vespaVersion, String dockerImageRepository) { - return deployApp(applicationPath, vespaVersion, Instant.now(), dockerImageRepository); + PrepareParams.Builder paramsBuilder = new PrepareParams.Builder(); + if (vespaVersion != null) + paramsBuilder.vespaVersion(vespaVersion); + + return deployApp(applicationPath, Instant.now(), paramsBuilder.dockerImageRepository(dockerImageRepository)); } /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ public PrepareResult deployApp(String applicationPath, String vespaVersion, Instant now) { - return deployApp(applicationPath, vespaVersion, now, null); + return deployApp(applicationPath, now, new PrepareParams.Builder().vespaVersion(vespaVersion)); } /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ - public PrepareResult deployApp(String applicationPath, String vespaVersion, Instant now, String dockerImageRepository) { - PrepareParams.Builder paramsBuilder = new PrepareParams.Builder() - .applicationId(applicationId) - .dockerImageRepository(dockerImageRepository) + public PrepareResult deployApp(String applicationPath, Instant now, PrepareParams.Builder paramsBuilder) { + paramsBuilder.applicationId(applicationId) .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60))); - if (vespaVersion != null) - paramsBuilder.vespaVersion(vespaVersion); return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build(), false, now); } @@ -299,8 +299,14 @@ public class DeployTester { } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - return hostProvisioner.prepare(cluster, capacity, groups, logger); + return hostProvisioner.prepare(cluster, capacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + return hostProvisioner.prepare(cluster, capacity, logger); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 5498d4b0315..7e700b78bf7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.model.TestModelFactory; +import com.yahoo.vespa.config.server.session.PrepareParams; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -87,17 +88,21 @@ public class HostedDeployTest { } @Test - public void testDeployWithWantedDockerImageRepository() throws IOException { + public void testReDeployWithWantedDockerImageRepositoryAndAthenzDomain() throws IOException { CountingModelFactory modelFactory = createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); DeployTester tester = new DeployTester(List.of(modelFactory), createConfigserverConfig()); String dockerImageRepository = "docker.foo.com:4443/bar/baz"; - tester.deployApp("src/test/apps/hosted/", "4.5.6", dockerImageRepository); + tester.deployApp("src/test/apps/hosted/", Instant.now(), new PrepareParams.Builder() + .vespaVersion("4.5.6") + .dockerImageRepository(dockerImageRepository) + .athenzDomain("foo")); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(tester.applicationId()); assertTrue(deployment.isPresent()); deployment.get().activate(); assertEquals("4.5.6", ((Deployment) deployment.get()).session().getVespaVersion().toString()); assertEquals(dockerImageRepository, ((Deployment) deployment.get()).session().getDockerImageRepository().get()); + assertEquals("foo", ((Deployment) deployment.get()).session().getAthenzDomain().get().value()); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index 70bba674778..5b0bb7885d8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -237,11 +237,17 @@ public class SessionHandlerTest { public Collection<HostSpec> lastHosts; @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { throw new UnsupportedOperationException(); } @Override + public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + throw new UnsupportedOperationException(); + } + + @Override public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { activated = true; lastApplicationId = application; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 67677822317..70f66cf8fde 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -28,12 +28,12 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import org.junit.After; import org.junit.Before; import org.junit.Test; import javax.ws.rs.client.Client; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -42,6 +42,7 @@ import java.nio.charset.StandardCharsets; import java.time.Clock; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; +import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -58,11 +59,16 @@ public class ApplicationHandlerTest { private static File testApp = new File("src/test/apps/app"); - private ListApplicationsHandler listApplicationsHandler; private final static TenantName mytenantName = TenantName.from("mytenant"); private final static TenantName foobar = TenantName.from("foobar"); - private final static ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); - + private final static ApplicationId myTenantApplicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); + private final static ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(TenantName.defaultName()).build(); + private final static MockTesterClient testerClient = new MockTesterClient(); + private final static NullMetric metric = new NullMetric(); + private final static ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); + private static final MockLogRetriever logRetriever = new MockLogRetriever(); + + private TestComponentRegistry componentRegistry; private TenantRepository tenantRepository; private ApplicationRepository applicationRepository; private SessionHandlerTest.MockProvisioner provisioner; @@ -71,27 +77,30 @@ public class ApplicationHandlerTest { @Before public void setup() { - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); + componentRegistry = new TestComponentRegistry.Builder().build(); tenantRepository = new TenantRepository(componentRegistry, false); - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName)); - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar)); provisioner = new SessionHandlerTest.MockProvisioner(); orchestrator = new OrchestratorMock(); applicationRepository = new ApplicationRepository(tenantRepository, provisioner, orchestrator, - new ConfigserverConfig(new ConfigserverConfig.Builder()), - new MockLogRetriever(), + configserverConfig, + logRetriever, Clock.systemUTC(), - new MockTesterClient(), - new NullMetric()); - listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), - tenantRepository, - Zone.defaultZone()); + testerClient, + metric); + } + + @After + public void shutdown() { + tenantRepository.close(); } @Test public void testDelete() throws Exception { + tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar)); + tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName)); + { applicationRepository.deploy(testApp, prepareParams(applicationId)); Tenant mytenant = tenantRepository.getTenant(applicationId.tenant()); @@ -132,7 +141,7 @@ public class ApplicationHandlerTest { @Test public void testDeleteNonExistent() throws Exception { - deleteAndAssertResponse(applicationId, + deleteAndAssertResponse(myTenantApplicationId, Zone.defaultZone(), Response.Status.NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, @@ -180,10 +189,10 @@ public class ApplicationHandlerTest { InfraDeployerProvider.empty(), new ConfigConvergenceChecker(stateApiFactory), mockHttpProxy, - new ConfigserverConfig(new ConfigserverConfig.Builder()), - new OrchestratorMock(), - new MockTesterClient(), - new NullMetric()); + configserverConfig, + orchestrator, + testerClient, + metric); ApplicationHandler mockHandler = createApplicationHandler(applicationRepository); when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1"))) .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>")); @@ -204,16 +213,15 @@ public class ApplicationHandlerTest { HttpResponse response = fileDistributionStatus(applicationId, zone); assertEquals(200, response.getStatus()); - SessionHandlerTest.getRenderedString(response); assertEquals("{\"hosts\":[{\"hostname\":\"mytesthost\",\"status\":\"UNKNOWN\",\"message\":\"error: Connection error(104)\",\"fileReferences\":[]}],\"status\":\"UNKNOWN\"}", - SessionHandlerTest.getRenderedString(response)); + getRenderedString(response)); // 404 for unknown application - ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant(mytenantName).build(); + ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant("default").build(); HttpResponse responseForUnknown = fileDistributionStatus(unknown, zone); assertEquals(404, responseForUnknown.getStatus()); - assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: 'mytenant.unknown'\"}", - SessionHandlerTest.getRenderedString(responseForUnknown)); + assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: 'default.unknown'\"}", + getRenderedString(responseForUnknown)); } @Test @@ -225,9 +233,7 @@ public class ApplicationHandlerTest { HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); assertEquals(200, response.getStatus()); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - response.render(baos); - assertEquals("log line", baos.toString()); + assertEquals("log line", getRenderedString(response)); } @Test @@ -235,13 +241,9 @@ public class ApplicationHandlerTest { applicationRepository.deploy(testApp, prepareParams(applicationId)); String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/status"; ApplicationHandler mockHandler = createApplicationHandler(); - HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); assertEquals(200, response.getStatus()); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - response.render(baos); - assertEquals("OK", baos.toString()); + assertEquals("OK", getRenderedString(response)); } @Test @@ -252,10 +254,7 @@ public class ApplicationHandlerTest { HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); assertEquals(200, response.getStatus()); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - response.render(baos); - assertEquals("log", baos.toString()); + assertEquals("log", getRenderedString(response)); } @Test @@ -345,10 +344,13 @@ public class ApplicationHandlerTest { "/environment/" + zone.environment().value() + "/region/" + zone.region().value() + "/instance/" + applicationId.instance().value() + "\"]"; + ListApplicationsHandler listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), + tenantRepository, + Zone.defaultZone()); ListApplicationsHandlerTest.assertResponse(listApplicationsHandler, "http://myhost:14000/application/v2/tenant/" + tenantName + "/application/", - Response.Status.OK, - expected, - com.yahoo.jdisc.http.HttpRequest.Method.GET); + Response.Status.OK, + expected, + com.yahoo.jdisc.http.HttpRequest.Method.GET); } private void restart(ApplicationId application, Zone zone) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 40115170b69..a3b0f3ec44a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -342,11 +342,17 @@ public class SessionPreparerTest { private static class FailWithTransientExceptionProvisioner implements Provisioner { @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception")); } @Override + public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception")); + } + + @Override public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { } @Override diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java index d7361eec488..bc7257b1ca9 100644 --- a/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java +++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java @@ -10,6 +10,8 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Paths; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; @@ -103,52 +105,46 @@ public class LogFileHandlerTestCase { h.shutdown(); } - @Test + @Test(timeout = /*5 minutes*/300_000) public void testSymlink() throws IOException, InterruptedException { File root = temporaryFolder.newFolder("testlogforsymlinkchecking"); - LogFileHandler h = new LogFileHandler(); - h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s"); - h.setFormatter(new Formatter() { + LogFileHandler handler = new LogFileHandler(); + handler.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s"); + handler.setFormatter(new Formatter() { public String format(LogRecord r) { DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS"); String timeStamp = df.format(new Date(r.getMillis())); return ("["+timeStamp+"]" + " " + formatMessage(r) + "\n"); } } ); - h.setSymlinkName("symlink"); - LogRecord lr = new LogRecord(Level.INFO, "test"); - h.publish(lr); - String f1 = h.getFileName(); - String f2 = null; - while (f1 == null) { - Thread.sleep(1); - f1 = h.getFileName(); - } - h.rotateNow(); - Thread.sleep(1); - f2 = h.getFileName(); - while (f1.equals(f2)) { + handler.setSymlinkName("symlink"); + + handler.publish(new LogRecord(Level.INFO, "test")); + String firstFile; + do { + Thread.sleep(1); + firstFile = handler.getFileName(); + } while (firstFile == null); + handler.rotateNow(); + String secondFileName; + do { Thread.sleep(1); - f2 = h.getFileName(); - } - lr = new LogRecord(Level.INFO, "string which is way longer than the word test"); - h.publish(lr); - h.waitDrained(); - File f = new File(f1); - long first = f.length(); - f = new File(f2); - long second = f.length(); - final long secondLength = 72; - for (int n = 0; n < 20 && second != secondLength; ++n) { + secondFileName = handler.getFileName(); + } while (firstFile.equals(secondFileName)); + + handler.publish(new LogRecord(Level.INFO, "string which is way longer than the word test")); + handler.waitDrained(); + assertThat(Files.size(Paths.get(firstFile))).isEqualTo(31); + final long expectedSecondFileLength = 72; + long secondFileLength; + do { Thread.sleep(1); - second = f.length(); - } - f = new File(root, "symlink"); - long link = f.length(); - assertThat(secondLength).isEqualTo(link); - assertThat(31).isEqualTo(first); - assertThat(secondLength).isEqualTo(second); - h.shutdown(); + secondFileLength = Files.size(Paths.get(secondFileName)); + } while (secondFileLength != expectedSecondFileLength); + + long symlinkFileLength = Files.size(root.toPath().resolve("symlink")); + assertThat(symlinkFileLength).isEqualTo(expectedSecondFileLength); + handler.shutdown(); } @Test diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index ce567176679..6d683c53984 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -250,6 +250,7 @@ "public void <init>(com.yahoo.container.QrSearchersConfig)", "public void <init>(com.yahoo.container.handler.ClustersStatus)", "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.handler.ClustersStatus)", + "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.core.VipStatusConfig, com.yahoo.container.handler.ClustersStatus, com.yahoo.container.jdisc.state.StateMonitor)", "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.handler.ClustersStatus, com.yahoo.container.jdisc.state.StateMonitor)", "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.core.VipStatusConfig, com.yahoo.container.handler.ClustersStatus)", "public void setInRotation(java.lang.Boolean)", diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java index 4c3d76436dd..ca11ad387ee 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java @@ -148,10 +148,9 @@ public class BundleLoader { /** * Returns the bundles that are not assumed to be retained by the new application generation. - * and cleans up the map of active file references. Note that at this point we don't yet know - * the full set of new bundles, because of the potential pre-install directives in the new bundles. - * However, only "disk bundles" (file:) can be listed in the pre-install directive, so we know - * about all the obsolete application bundles. + * Note that at this point we don't yet know the full set of new bundles, because of the potential + * pre-install directives in the new bundles. However, only "disk bundles" (file:) can be listed + * in the pre-install directive, so we know about all the obsolete application bundles. */ private Set<Bundle> getObsoleteBundles(List<FileReference> newReferences) { Set<Bundle> bundlesToRemove = new HashSet<>(osgi.getCurrentBundles()); @@ -165,6 +164,9 @@ public class BundleLoader { return bundlesToRemove; } + /** + * Cleans up the map of active file references + */ private void removeInactiveFileReferences(List<FileReference> newReferences) { // Clean up the map of active bundles Set<FileReference> fileReferencesToRemove = getObsoleteFileReferences(newReferences); @@ -184,6 +186,7 @@ public class BundleLoader { // The bundle at index 0 for each file reference always corresponds to the bundle at the file reference location Set<Bundle> allowedDuplicates = obsoleteReferences.stream() + .filter(reference -> ! isDiskBundle(reference)) .map(reference -> reference2Bundles.get(reference).get(0)) .collect(Collectors.toSet()); diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index f712690efc5..0bf86e8f440 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -24,6 +24,8 @@ public class VipStatus { /** If this is non-null, its value decides whether this container is in rotation */ private Boolean rotationOverride = null; + private final boolean initiallyInRotation; + /** The current state of this */ private boolean currentlyInRotation; @@ -44,20 +46,29 @@ public class VipStatus { this(new QrSearchersConfig.Builder().build(), clustersStatus); } + /** For testing */ public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus) { - this(dispatchers, clustersStatus, new StateMonitor()); + this(dispatchers, new VipStatusConfig.Builder().build(), clustersStatus, new StateMonitor()); } @Inject - public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus, StateMonitor healthState) { + public VipStatus(QrSearchersConfig dispatchers, + VipStatusConfig vipStatusConfig, + ClustersStatus clustersStatus, + StateMonitor healthState) { this.clustersStatus = clustersStatus; this.healthState = healthState; + initiallyInRotation = vipStatusConfig.initiallyInRotation(); healthState.status(StateMonitor.Status.initializing); clustersStatus.setContainerHasClusters(! dispatchers.searchcluster().isEmpty()); updateCurrentlyInRotation(); } - /** @deprecated don't pass VipStatusConfig */ + @Deprecated // TODO: Remove on Vespa 8 + public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus, StateMonitor healthState) { + this(dispatchers, new VipStatusConfig.Builder().build(), clustersStatus, healthState); + } + @Deprecated // TODO: Remove on Vespa 8 public VipStatus(QrSearchersConfig dispatchers, VipStatusConfig ignored, ClustersStatus clustersStatus) { this(dispatchers, clustersStatus); @@ -107,7 +118,12 @@ public class VipStatus { } else { if (healthState.status() == StateMonitor.Status.up) { currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ONE); - } else { + } + else if (healthState.status() == StateMonitor.Status.initializing) { + currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ALL) + && initiallyInRotation; + } + else { currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ALL); } } diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java index d3479936544..e13debcddda 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.container.handler; import static org.junit.Assert.*; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.jdisc.core.SystemTimer; import org.junit.Test; @@ -14,39 +15,45 @@ import org.junit.Test; * @author steinar */ public class VipStatusTestCase { - private static final String [] clusters = {"cluster1", "cluster2", "cluster3"}; - private static QrSearchersConfig getSearchersCfg() { + private static QrSearchersConfig getSearchersConfig(String[] clusters) { var b = new QrSearchersConfig.Builder(); - var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); - for (String cluster : clusters) { - searchClusterB.name(cluster); + if (clusters.length > 0) { + var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); + for (String cluster : clusters) { + searchClusterB.name(cluster); + } + b.searchcluster(searchClusterB); } - b.searchcluster(searchClusterB); return b.build(); } - private static VipStatus getVipStatus(StateMonitor.Status startState) { - return new VipStatus(getSearchersCfg(), new ClustersStatus(), new StateMonitor(1000, startState, new SystemTimer(), runnable -> { - Thread thread = new Thread(runnable, "StateMonitor"); - thread.setDaemon(true); - return thread; - })); + + private static VipStatus getVipStatus(String[] clusters, StateMonitor.Status startState, boolean initiallyInRotation) { + return new VipStatus(getSearchersConfig(clusters), + new VipStatusConfig.Builder().initiallyInRotation(initiallyInRotation).build(), + new ClustersStatus(), + new StateMonitor(1000, startState, new SystemTimer(), runnable -> { + Thread thread = new Thread(runnable, "StateMonitor"); + thread.setDaemon(true); + return thread; + })); } - private static void removeAll(VipStatus v) { + private static void remove(String[] clusters, VipStatus v) { for (String s : clusters) { v.removeFromRotation(s); } } - private static void addAll(VipStatus v) { + + private static void add(String[] clusters, VipStatus v) { for (String s : clusters) { v.addToRotation(s); } } - private static void verifyUpOrDown(StateMonitor.Status status) { - VipStatus v = getVipStatus(status); - removeAll(v); + private static void verifyUpOrDown(String[] clusters, StateMonitor.Status status) { + VipStatus v = getVipStatus(clusters, status, true); + remove(clusters, v); // initial state assertFalse(v.isInRotation()); v.addToRotation(clusters[0]); @@ -59,15 +66,18 @@ public class VipStatusTestCase { @Test public void testInitializingOrDownRequireAllUp() { - verifyUpOrDown(StateMonitor.Status.initializing); - verifyUpOrDown(StateMonitor.Status.down); + String[] clusters = {"cluster1", "cluster2", "cluster3"}; + verifyUpOrDown(clusters, StateMonitor.Status.initializing); + verifyUpOrDown(clusters, StateMonitor.Status.down); } @Test public void testUpRequireAllDown() { - VipStatus v = getVipStatus(StateMonitor.Status.initializing); + String[] clusters = {"cluster1", "cluster2", "cluster3"}; + + VipStatus v = getVipStatus(clusters, StateMonitor.Status.initializing, true); assertFalse(v.isInRotation()); - addAll(v); + add(clusters, v); assertTrue(v.isInRotation()); v.removeFromRotation(clusters[0]); @@ -89,4 +99,18 @@ public class VipStatusTestCase { assertTrue(v.isInRotation()); } -} + @Test + public void testNoClustersConfiguringInitiallyInRotationFalse() { + String[] clusters = {}; + VipStatus v = getVipStatus(clusters, StateMonitor.Status.initializing, false); + assertFalse(v.isInRotation()); + } + + @Test + public void testNoClustersConfiguringInitiallyInRotationTrue() { + String[] clusters = {}; + VipStatus v = getVipStatus(clusters, StateMonitor.Status.initializing, true); + assertTrue(v.isInRotation()); + } + +}
\ No newline at end of file diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index 113d99f77f9..7193433ccf7 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -120,7 +120,10 @@ public final class SessionCache extends AbstractComponent { RPCNetworkParams netParams = new RPCNetworkParams() .setSlobrokConfigId(slobrokConfigId) .setIdentity(new Identity(identity)) - .setListenPort(mbusConfig.port()); + .setListenPort(mbusConfig.port()) + .setNumTargetsPerSpec(mbusConfig.numconnectionspertarget()) + .setNumNetworkThreads(mbusConfig.numthreads()) + .setOptimization(RPCNetworkParams.Optimization.valueOf(mbusConfig.optimize_for().name())); return SharedMessageBus.newInstance(mbusParams, netParams); } diff --git a/container-messagebus/src/main/resources/configdefinitions/container-mbus.def b/container-messagebus/src/main/resources/configdefinitions/container-mbus.def index b18bec66959..9aef2b32a66 100644 --- a/container-messagebus/src/main/resources/configdefinitions/container-mbus.def +++ b/container-messagebus/src/main/resources/configdefinitions/container-mbus.def @@ -2,9 +2,24 @@ namespace=container.jdisc #settings for message bus in container -enabled bool default=false + +# Which network port is used port int default=0 + +# Number of connections per target +numconnectionspertarget int default=1 + +# Number network threads +numthreads int default=2 + +# Optimize for latency, or throughput. +optimize_for enum {LATENCY, THROUGHPUT} default=LATENCY + +# Everying below is deprecated and will go away very soon. +# Dynamic throttling is used, and works better than anything else. maxpendingcount int default=2048 + +enabled bool default=false #maxpendingsize is set in megabytes! maxpendingsize int default=100 diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index 3d244312b2f..0686a4bdb43 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -126,7 +126,6 @@ abstract class SimpleParser extends StructuredParser { if (topLevelItem != null && topLevelItem != not) { // => neutral rank items becomes implicit positives - System.out.println("Extracting positive item from " + topLevelItem); not.addPositiveItem(getItemAsPositiveItem(topLevelItem, not)); return not; } else { // Only negatives - ignore them diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 2d05168731a..645c6446ef1 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -220,14 +220,6 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if (result == null) result = new Result(query, ErrorMessage.createBackendCommunicationError("No result returned in " + this + " from " + connection + " for " + query)); - - if (result.hits().getError() != null) { - log(LogLevel.FINE, "FAILED: ", query); - } else if ( ! result.isCached()) { - log(LogLevel.FINE, "WORKING: ", query); - } else { - log(LogLevel.FINE, "CACHE HIT: ", query); - } return result; } @@ -263,13 +255,6 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod result.hits().addError(ErrorMessage.createBackendCommunicationError("Error filling " + result + " from " + connection + ": " + Exceptions.toMessageString(e))); } - if (result.hits().getError() != null) { - log(LogLevel.FINE, "FAILED: ", result.getQuery()); - } else if ( ! result.isCached()) { - log(LogLevel.FINE, "WORKING: ", result.getQuery()); - } else { - log(LogLevel.FINE, "CACHE HIT: " + result.getQuery()); - } } /** diff --git a/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java b/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java index 46752b0bedb..6c83e1c64e3 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java @@ -11,6 +11,7 @@ package com.yahoo.search.cluster; public class Hasher<T> { public static class NodeFactor<T> { + private final T node; /** diff --git a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java index 21e5fe3bc7f..a2fb982e3c5 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java @@ -8,62 +8,43 @@ package com.yahoo.search.cluster; */ public class MonitorConfiguration { - /** - * The interval in ms between consecutive checks of the monitored - * nodes - */ + /** The interval in ms between consecutive checks of the monitored nodes */ private long checkInterval=1000; - /** - * The number of milliseconds to attempt to complete a request - * before giving up - */ + /** The number of milliseconds to attempt to complete a request before giving up */ private final long requestTimeout = 980; - /** - * The number of milliseconds a node is allowed to fail before we - * mark it as not working - */ - private long failLimit=5000; + /** The number of milliseconds a node is allowed to fail before we mark it as not working */ + private long failLimit = 5000; - /** - * Sets the interval between each ping of idle or failing nodes - * Default is 1000ms - */ - public void setCheckInterval(long intervalMs) { - this.checkInterval=intervalMs; - } + /** Sets the interval between each ping of idle or failing nodes. Default is 1000 ms. */ + public void setCheckInterval(long intervalMs) { this.checkInterval = intervalMs; } - /** - * Returns the interval between each ping of idle or failing nodes - * Default is 1000ms - */ - public long getCheckInterval() { - return checkInterval; - } + /** Returns the interval between each ping of idle or failing nodes. Default is 1000 ms. */ + public long getCheckInterval() { return checkInterval; } /** - * Sets the number of times a failed node must respond before it is put - * back in service. Default is 3. - * @deprecated Will go away in Vespa 8 + * Sets the number of times a failed node must respond before it is put back in service. Default is 3. + * + * @deprecated will go away in Vespa 8 */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 public void setResponseAfterFailLimit(int responseAfterFailLimit) { } /** - * Sets the number of ms a node (failing or working) is allowed to - * stay idle before it is pinged. Default is 3000 + * Sets the number of ms a node (failing or working) is allowed to stay idle before it is pinged. Default is 3000. + * * @deprecated Will go away in Vespa 8 */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 public void setIdleLimit(int idleLimit) { } /** - * Gets the number of ms a node (failing or working) - * is allowed to stay idle before it is pinged. Default is 3000 + * Gets the number of ms a node (failing or working) is allowed to stay idle before it is pinged. Default is 3000. + * * @deprecated Will go away in Vespa 8 */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 public long getIdleLimit() { return 3000; } @@ -91,25 +72,26 @@ public class MonitorConfiguration { * in quarantine. Once in quarantine it won't be put back in * productuion before quarantineTime has expired even if it is * working. Default is 3 + * * @deprecated Will go away in Vespa 8 */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 public void setFailQuarantineLimit(int failQuarantineLimit) { } /** - * The number of ms an unstable node is quarantined. Default is - * 100*60*60 + * The number of ms an unstable node is quarantined. Default is 100*60*60 + * * @deprecated Will go away in Vespa 8 */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 public void setQuarantineTime(long quarantineTime) { } public String toString() { return "monitor configuration [" + - "checkInterval: " + checkInterval + - " requestTimeout " + requestTimeout + - " failLimit " + failLimit + - "]"; + "checkInterval: " + checkInterval + + " requestTimeout " + requestTimeout + + " failLimit " + failLimit + + "]"; } } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java index 481f1e1b5a5..836c71089c1 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java @@ -7,7 +7,7 @@ import java.util.concurrent.Executor; * Must be implemented by a node collection which wants * it's node state monitored by a ClusterMonitor * - * @author bratseth + * @author bratseth */ public interface NodeManager<T> { @@ -20,9 +20,10 @@ public interface NodeManager<T> { /** * Called when a node should be pinged. * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded + * * @deprecated Use ping(ClusterMonitor clusterMonitor, T node, Executor executor) instead. */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 default void ping(T node, Executor executor) { throw new IllegalStateException("If you have not overrriden ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor), you should at least have overriden this method."); } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java index ccf3e863ff3..d17f6bfbaa8 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java @@ -52,8 +52,8 @@ public class TrafficNodeMonitor<T> extends BaseNodeMonitor<T> { * Called when a response is received from this node. */ public void responded() { - respondedAt=now(); - succeededAt=respondedAt; + respondedAt = now(); + succeededAt = respondedAt; setWorking(true,"Responds correctly"); } @@ -69,20 +69,20 @@ public class TrafficNodeMonitor<T> extends BaseNodeMonitor<T> { atStartUp = false; if (this.isWorking == working) return; // Old news - if (explanation==null) { - explanation=""; + if (explanation == null) { + explanation = ""; } else { - explanation=": " + explanation; + explanation = ": " + explanation; } if (working) { log.info("Putting " + node + " in service" + explanation); } else { log.warning("Taking " + node + " out of service" + explanation); - failedAt=now(); + failedAt = now(); } - this.isWorking=working; + this.isWorking = working; } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 7619cb34b77..7862648ba51 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -78,10 +78,10 @@ public class SearchCluster implements NodeManager<Node> { this.nodesByHost = nodesByHostBuilder.build(); this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), - size, - containerClusterSize, - nodesByHost, - groups); + size, + containerClusterSize, + nodesByHost, + groups); } /* Testing only */ @@ -217,7 +217,10 @@ public class SearchCluster implements NodeManager<Node> { setInRotationOnlyIf(hasWorkingNodes()); } else if (usesLocalCorpusIn(node)) { // follow the status of this node - setInRotationOnlyIf(nodeIsWorking); + // Do not take this out of rotation if we're a combined cluster of size 1, + // as that can't be helpful, and leads to a deadlock where this node is never taken back in servic e + if (nodeIsWorking || size() > 1) + setInRotationOnlyIf(nodeIsWorking); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java index cb662dcd671..65ffd29efe0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Select.java +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -57,12 +57,13 @@ public class Select implements Cloneable { } public Select(String where, String grouping, Query query) { - this(where, grouping, query, Collections.emptyList()); + this(where, grouping, null, query, Collections.emptyList()); } - private Select(String where, String grouping, Query query, List<GroupingRequest> groupingRequests) { + private Select(String where, String grouping, String groupingExpressionString, Query query, List<GroupingRequest> groupingRequests) { this.where = Objects.requireNonNull(where, "A Select must have a where string (possibly the empty string)"); this.grouping = Objects.requireNonNull(grouping, "A Select must have a select string (possibly the empty string)"); + this.groupingExpressionString = groupingExpressionString; this.parent = Objects.requireNonNull(query, "A Select must have a parent query"); this.groupingRequests = deepCopy(groupingRequests, this); } @@ -136,11 +137,11 @@ public class Select implements Cloneable { @Override public Object clone() { - return new Select(where, grouping, parent, groupingRequests); + return new Select(where, grouping, groupingExpressionString, parent, groupingRequests); } public Select cloneFor(Query parent) { - return new Select(where, grouping, parent, groupingRequests); + return new Select(where, grouping, groupingExpressionString, parent, groupingRequests); } } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index cf90a1c6d81..ad281aeda7d 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -191,7 +191,7 @@ public class SearchClusterTest { } @Test - public void requireThatVipStatusIsDefaultDownWithOnlySingleLocalDispatch() { + public void requireThatVipStatusStaysUpWithLocalDispatchAndClusterSize1() { try (State test = new State("cluster.1", 1, HostName.getLocalhost())) { assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); @@ -200,6 +200,20 @@ public class SearchClusterTest { assertTrue(test.vipStatus.isInRotation()); test.numDocsPerNode.get(0).set(-1); test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + } + } + + @Test + public void requireThatVipStatusIsDefaultDownWithLocalDispatchAndClusterSize2() { + try (State test = new State("cluster.1", 1, HostName.getLocalhost(), "otherhost")) { + assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); + + assertFalse(test.vipStatus.isInRotation()); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(0).set(-1); + test.waitOneFullPingRound(); assertFalse(test.vipStatus.isInRotation()); } } diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 7b1b4fe6362..1715ed38964 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -724,6 +724,7 @@ public class SelectTestCase { assertEquals("all(group(time.dayofmonth(a)) each(output(count())))", query.getSelect().getGrouping().get(0).toString()); Query clone = query.clone(); + assertEquals(clone.getSelect().getGroupingExpressionString(), query.getSelect().getGroupingExpressionString()); assertNotSame(query.getSelect(), clone.getSelect()); assertNotSame(query.getSelect().getGrouping(), clone.getSelect().getGrouping()); assertNotSame(query.getSelect().getGrouping().get(0), clone.getSelect().getGrouping().get(0)); @@ -732,8 +733,15 @@ public class SelectTestCase { assertEquals(query.getSelect().getGroupingString(), clone.getSelect().getGroupingString()); assertEquals(query.getSelect().getGrouping().get(0).toString(), clone.getSelect().getGrouping().get(0).toString()); assertEquals(query.getSelect().getGrouping().get(1).toString(), clone.getSelect().getGrouping().get(1).toString()); + } + @Test + public void testCloneWithGroupingExpressionString() { + Query query = new Query(); + query.getSelect().setGroupingExpressionString("all(group(foo) each(output(count())))"); + Query clone = query.clone(); + assertEquals(clone.getSelect().getGroupingExpressionString(), query.getSelect().getGroupingExpressionString()); } //------------------------------------------------------------------- Assert methods diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java index 4974192e213..3ac24bac7ca 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java @@ -10,10 +10,6 @@ public class TenantId extends NonDefaultIdentifier { super(id); } - public boolean isUser() { - return id().startsWith("by-"); - } - @Override public void validate() { super.validate(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java index d2effc76827..f1a8e57ab03 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java @@ -10,8 +10,4 @@ public class UserId extends NonDefaultIdentifier { super(id); } - public TenantId toTenantId() { - return new TenantId("by-" + id().replace('_', '-')); - } - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index bdd2fb247fa..5c11dfc2a55 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -58,7 +58,8 @@ enum PathGroup { "/application/v4/tenant/{tenant}/application/", "/application/v4/tenant/{tenant}/cost", "/application/v4/tenant/{tenant}/cost/{date}", - "/routing/v1/status/tenant/{tenant}/{*}"), + "/routing/v1/status/tenant/{tenant}/{*}", + "/billing/v1/tenant/{tenant}/{*}"), tenantKeys(Matcher.tenant, PathPrefix.api, @@ -201,7 +202,11 @@ enum PathGroup { /** Paths used for "dry-running" system-wide feature flags. */ - systemFlagsDryrun(PathPrefix.none, "/system-flags/v1/dryrun"); + systemFlagsDryrun(PathPrefix.none, "/system-flags/v1/dryrun"), + + /** Paths used for receiving payment callbacks */ + paymentProcessor(PathPrefix.none, "/payment/notification"); + final List<String> pathSpecs; final PathPrefix prefix; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index 55512b38f95..cfe8d247e54 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -137,7 +137,12 @@ enum Policy { /** Access to /system-flags/v1/dryrun. */ systemFlagsDryrun(Privilege.grant(Action.update) .on(PathGroup.systemFlagsDryrun) - .in(SystemName.all())); + .in(SystemName.all())), + + /** Access to /payment/notification */ + paymentProcessor(Privilege.grant(Action.create) + .on(PathGroup.paymentProcessor) + .in(SystemName.PublicCd)); private final Set<Privilege> privileges; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java index 532088e94aa..d3c5e412215 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java @@ -73,6 +73,9 @@ public abstract class Role { /** Returns the role for system flag dryrun */ public static UnboundRole systemFlagsDryrunner() { return new UnboundRole(RoleDefinition.systemFlagsDryrunner); } + /** Returns the role of the payment processor */ + public static UnboundRole paymentProcessor() { return new UnboundRole(RoleDefinition.paymentProcessor); } + /** Returns the role definition of this bound role. */ public RoleDefinition definition() { return roleDefinition; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index c4ce70a8f1e..c05936ee593 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -76,7 +76,9 @@ public enum RoleDefinition { systemFlagsDeployer(Policy.systemFlagsDeploy, Policy.systemFlagsDryrun), - systemFlagsDryrunner(Policy.systemFlagsDryrun); + systemFlagsDryrunner(Policy.systemFlagsDryrun), + + paymentProcessor(Policy.paymentProcessor); private final Set<RoleDefinition> parents; private final Set<Policy> policies; diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java index 8e278240a02..fdba1ab2680 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java @@ -107,11 +107,6 @@ public class IdentifierTest { } @Test - public void user_tenant_id_does_not_contain_underscore() { - assertEquals("by-under-score-user", new UserId("under_score_user").toTenantId().id()); - } - - @Test public void dns_names_has_no_underscore() { assertEquals("a-b-c", new ApplicationId("a_b_c").toDns()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 2b4b251f536..c3a9a8484c9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -243,6 +243,10 @@ public class AthenzFacade implements AccessControl { return hasAccess(dryRun ? "dryrun" : "deploy", new AthenzResourceName(service.getDomain(), "system-flags").toResourceNameString(), identity); } + public boolean hasPaymentCallbackAccess(AthenzIdentity identity) { + return hasAccess("callback", new AthenzResourceName(service.getDomain().getName(), "payment-notification-resource").toResourceNameString(), identity); + } + /** * Used when creating tenancies. As there are no tenancy policies at this point, * we cannot use {@link #hasTenantAdminAccess(AthenzIdentity, AthenzDomain)} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 596e19bfe65..71be06deec5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -941,8 +941,8 @@ public class InternalStepRunner implements StepRunner { Duration endpoint() { return Duration.ofMinutes(15); } Duration endpointCertificate() { return Duration.ofMinutes(15); } Duration tester() { return Duration.ofMinutes(30); } - Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 60); } - Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 120); } + Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 20 : 60); } + Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 20 : 120); } Duration testerCertificate() { return Duration.ofMinutes(300); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java index e3cebfad31c..921bf045873 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java @@ -53,7 +53,8 @@ public class NodeWithServices { } public boolean needsPlatformUpgrade() { - return node.wantedVersion().isAfter(node.currentVersion()); + return node.wantedVersion().isAfter(node.currentVersion()) + || ! node.wantedDockerImage().equals(node.currentDockerImage()); } public boolean needsReboot() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index be3f4e50dc7..d7ad96ec5e7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -29,7 +29,6 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -106,7 +105,6 @@ import java.time.Duration; import java.time.Instant; import java.time.YearMonth; import java.time.format.DateTimeParseException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Comparator; @@ -205,7 +203,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handleGET(Path path, HttpRequest request) { if (path.matches("/application/v4/")) return root(request); - if (path.matches("/application/v4/user")) return authenticatedUser(request); if (path.matches("/application/v4/tenant")) return tenants(request); if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/cost")) return tenantCost(path.get("tenant"), request); @@ -248,7 +245,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse handlePUT(Path path, HttpRequest request) { - if (path.matches("/application/v4/user")) return new EmptyResponse(); if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); @@ -325,24 +321,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse root(HttpRequest request) { return recurseOverTenants(request) ? recursiveRoot(request) - : new ResourceResponse(request, "user", "tenant"); - } - - // TODO jonmv: Move to Athenz API. - private HttpResponse authenticatedUser(HttpRequest request) { - Principal user = requireUserPrincipal(request); - - String userName = user instanceof AthenzPrincipal ? ((AthenzPrincipal) user).getIdentity().getName() : user.getName(); - List<Tenant> tenants = controller.tenants().asList(new Credentials(user)); - - Slime slime = new Slime(); - Cursor response = slime.setObject(); - response.setString("user", userName); - Cursor tenantsArray = response.setArray("tenants"); - for (Tenant tenant : tenants) - tenantInTenantsListToSlime(tenant, request.getUri(), tenantsArray.addObject()); - response.setBool("tenantExists", true); - return new SlimeJsonResponse(slime); + : new ResourceResponse(request, "tenant"); } private HttpResponse tenants(HttpRequest request) { @@ -1062,12 +1041,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Add zone endpoints var endpointArray = response.setArray("endpoints"); - var serviceUrls = new ArrayList<URI>(); for (var endpoint : controller.routing().endpointsOf(deploymentId)) { toSlime(endpoint, endpoint.name(), endpointArray.addObject()); - if (endpoint.routingMethod() == RoutingMethod.shared) { - serviceUrls.add(endpoint.url()); - } } // Add global endpoints var globalEndpoints = controller.routing().endpointsOf(application, deploymentId.applicationId().instance()) @@ -1077,9 +1052,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // TODO(mpolden): Pass cluster name. Cluster that a global endpoint points to is not available at this level. toSlime(endpoint, "", endpointArray.addObject()); } - // TODO(mpolden): Remove this once all clients stop reading it - Cursor serviceUrlArray = response.setArray("serviceUrls"); - serviceUrls.forEach(url -> serviceUrlArray.addString(url.toString())); response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index afe8d156d00..4d3cfacab14 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -107,7 +107,8 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { if (identity.getDomain().equals(SCREWDRIVER_DOMAIN) && application.isPresent() && tenant.isPresent()) futures.add(executor.submit(() -> { - if (hasDeployerAccess(identity, ((AthenzTenant) tenant.get()).domain(), application.get())) + if ( tenant.get().type() == Tenant.Type.athenz + && hasDeployerAccess(identity, ((AthenzTenant) tenant.get()).domain(), application.get())) roleMemberships.add(Role.buildService(tenant.get().name(), application.get())); })); @@ -116,6 +117,11 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { roleMemberships.add(Role.systemFlagsDeployer()); })); + futures.add(executor.submit(() -> { + if (athenz.hasPaymentCallbackAccess(identity)) + roleMemberships.add(Role.paymentProcessor()); + })); + // Run last request in handler thread to avoid creating extra thread. if (athenz.hasSystemFlagsAccess(identity, /*dryrun*/true)) roleMemberships.add(Role.systemFlagsDryrunner()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 497a0ddf5a0..351f530f623 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -788,6 +788,7 @@ public class ControllerTest { @Test public void testDeployWithRoutingGeneratorEndpoints() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.DISABLE_ROUTING_GENERATOR.id(), false); var context = tester.newDeploymentContext(); var applicationPackage = new ApplicationPackageBuilder() .upgradePolicy("default") @@ -804,6 +805,7 @@ public class ControllerTest { List.of(new RoutingEndpoint("http://legacy-endpoint", "hostname", false, "upstreamName"))); } + // Defer load balancer provisioning in all environments so that routing controller uses routing generator context.deferLoadBalancerProvisioningIn(zones.stream().map(ZoneId::environment).collect(Collectors.toSet())) .submit(applicationPackage) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 204e0a1c1b8..9e9d27fd744 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -6,6 +6,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; @@ -72,6 +73,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Map<DeploymentId, ServiceConvergence> serviceStatus = new HashMap<>(); private final Set<ApplicationId> disallowConvergenceCheckApplications = new HashSet<>(); private final Version initialVersion = new Version(6, 1, 0); + private final DockerImage initialDockerImage = DockerImage.fromString("dockerImage:6.1.0"); private final Set<DeploymentId> suspendedApplications = new HashSet<>(); private final Map<ZoneId, Set<LoadBalancer>> loadBalancers = new HashMap<>(); private final Set<Environment> deferLoadBalancerProvisioning = new HashSet<>(); @@ -105,6 +107,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer .parentHostname(parent.hostname()) .currentVersion(initialVersion) .wantedVersion(initialVersion) + .currentDockerImage(initialDockerImage) + .wantedDockerImage(initialDockerImage) .currentOsVersion(Version.emptyVersion) .wantedOsVersion(Version.emptyVersion) .resources(new NodeResources(2, 8, 50, 1, slow, remote)) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 632b8499e11..4aab21a44fe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -208,7 +208,10 @@ public class NodeRepositoryMock implements NodeRepository { public void doUpgrade(DeploymentId deployment, Optional<HostName> hostName, Version version) { modifyNodes(deployment, hostName, node -> { assert node.wantedVersion().equals(version); - return new Node.Builder(node).currentVersion(version).build(); + return new Node.Builder(node) + .currentVersion(version) + .currentDockerImage(node.wantedDockerImage()) + .build(); }); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index fd0981e8427..2752ba64b61 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -176,24 +176,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); - // GET the authenticated user (with associated tenants) - tester.assertResponse(request("/application/v4/user", GET).userIdentity(USER_ID), - new File("user.json")); - // TODO jonmv: Remove when dashboard is gone. - // PUT a user tenant — does nothing - tester.assertResponse(request("/application/v4/user", PUT).userIdentity(USER_ID), - ""); - - // GET the authenticated user which now exists (with associated tenants) - tester.assertResponse(request("/application/v4/user", GET).userIdentity(USER_ID), - new File("user.json")); - - // DELETE the user — it doesn't exist, so access control fails - tester.assertResponse(request("/application/v4/tenant/by-myuser", DELETE).userIdentity(USER_ID), - "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", 403); - // GET all tenants - tester.assertResponse(request("/application/v4/tenant/", GET).userIdentity(USER_ID), - new File("tenant-list.json")); // GET list of months for a tenant tester.assertResponse(request("/application/v4/tenant/tenant1/cost", GET).userIdentity(USER_ID).oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), @@ -783,11 +765,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "{\"message\":\"Aborting run 2 of staging-test for tenant1.application1.instance1\"}"); - // GET user lists only tenants for the authenticated user - tester.assertResponse(request("/application/v4/user", GET) - .userIdentity(new UserId("other_user")), - "{\"user\":\"other_user\",\"tenants\":[],\"tenantExists\":true}"); - // OPTIONS return 200 OK tester.assertResponse(request("/application/v4/", Request.Method.OPTIONS) .userIdentity(USER_ID), @@ -1108,14 +1085,6 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"error-code\":\"BAD_REQUEST\",\"message\":\"New tenant or application names must start with a letter, may contain no more than 20 characters, and may only contain lowercase letters, digits or dashes, but no double-dashes.\"}", 400); - // POST (add) an Athenz tenant with by- prefix - tester.assertResponse(request("/application/v4/tenant/by-tenant2", POST) - .userIdentity(USER_ID) - .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Athenz tenant name cannot have prefix 'by-'\"}", - 400); - // POST (add) an Athenz tenant with a reserved name tester.assertResponse(request("/application/v4/tenant/hosted-vespa", POST) .userIdentity(USER_ID) @@ -1395,25 +1364,12 @@ public class ApplicationApiTest extends ControllerContainerTest { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, tenantAdmin); allowLaunchOfService(new com.yahoo.vespa.athenz.api.AthenzService(ATHENZ_TENANT_DOMAIN, "service")); - // Create tenant - // PUT (create) the authenticated user - tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT) - .userIdentity(userId), // Normalized to by-new-user by API - ""); - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain1"), com.yahoo.config.provision.AthenzService.from("service")) .build(); - // POST (deploy) an application to a dev zone fails because user tenant is used — these do not exist. - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); - tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-west-1/instance/default", POST) - .data(entity) - .userIdentity(userId), - "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", - 403); - createTenantAndApplication(); + MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); // POST (deploy) an application to dev through a deployment job, with user instance and a proper tenant tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST) .data(entity) @@ -1426,13 +1382,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .domains.get(ATHENZ_TENANT_DOMAIN) .admin(HostedAthenzIdentities.from(userId)); - // POST (deploy) an application to a dev zone fails because user tenant is used — these do not exist. - tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-west-1/instance/default", POST) - .data(entity) - .userIdentity(userId), - "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", - 403); - // POST (deploy) an application to dev through a deployment job tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST) .data(entity) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index cd47859c7cc..63e6e4b3937 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -20,9 +20,6 @@ "routingMethod": "shared" } ], - "serviceUrls": [ - "https://instance1--application1--tenant1.us-west-1.vespa.oath.cloud:4443/" - ], "nodes": "http://localhost:8080/zone/v2/prod/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-west-1&application=tenant1.application1.instance1", "version": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 726df575028..928525a20d1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -20,9 +20,6 @@ "routingMethod": "shared" } ], - "serviceUrls": [ - "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/" - ], "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", "version": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 7c231beb5ed..83fa1983957 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -13,9 +13,6 @@ "routingMethod": "shared" } ], - "serviceUrls": [ - "https://instance1--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/" - ], "nodes": "http://localhost:8080/zone/v2/dev/us-east-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-east-1&application=tenant1.application1.instance1", "version": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 41f3908f12f..4ffe809297d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -23,9 +23,6 @@ "routingMethod": "shared" } ], - "serviceUrls": [ - "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/" - ], "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", "version": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json index 986245decca..d63a7ba7d56 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json @@ -1,9 +1,6 @@ { "resources":[ { - "url":"http://localhost:8080/application/v4/user/" - }, - { "url":"http://localhost:8080/application/v4/tenant/" } ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user-which-exists.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user-which-exists.json deleted file mode 100644 index f2703677738..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user-which-exists.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "user": "myuser", - "tenants": @include(tenant-list-with-user.json), - "tenantExists": true -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json deleted file mode 100644 index 9902267dbb5..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "user": "myuser", - "tenants": @include(tenant-list.json), - "tenantExists": true -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java index c49f7a90194..5e50e80b7a7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java @@ -70,13 +70,13 @@ public class AthenzRoleFilterTest { public void testTranslations() throws Exception { // Hosted operators are always members of the hostedOperator role. - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.hostedSupporter()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, NO_CONTEXT_PATH)); - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.hostedSupporter()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, TENANT_CONTEXT_PATH)); - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.hostedSupporter()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, APPLICATION_CONTEXT_PATH)); // Tenant admins are members of the athenzTenantAdmin role within their tenant subtree. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 6db5bc9f523..51466e5b1e2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -71,11 +71,6 @@ public class UserApiTest extends ControllerContainerCloudTest { .data("{\"token\":\"hello\"}"), new File("tenant-without-applications.json")); - // PUT a tenant is ignored. - tester.assertResponse(request("/application/v4/user/", PUT) - .roles(operator), - "", 200); - // GET at user/v1 root fails as no access control is defined there. tester.assertResponse(request("/user/v1/"), accessDenied, 403); diff --git a/dist/vespa.spec b/dist/vespa.spec index 4281ce243fa..c89cd42df0f 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -8,6 +8,7 @@ %define _vespa_deps_prefix /opt/vespa-deps %define _vespa_user vespa %define _vespa_group vespa +%undefine _vespa_user_uid %define _create_vespa_group 1 %define _create_vespa_user 1 %define _create_vespa_service 1 @@ -103,7 +104,7 @@ BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %endif -BuildRequires: xxhash-devel >= 0.6.5 +BuildRequires: xxhash-devel >= 0.7.3 BuildRequires: openblas-devel BuildRequires: lz4-devel BuildRequires: libzstd-devel @@ -143,7 +144,7 @@ Requires: perl-URI Requires: valgrind Requires: Judy Requires: xxhash -Requires: xxhash-libs >= 0.6.5 +Requires: xxhash-libs >= 0.7.3 %if 0%{?el8} Requires: openblas %else @@ -207,17 +208,107 @@ Requires: llvm-libs >= 10.0.0 %define _extra_link_directory %{_vespa_deps_prefix}/lib64 %define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas %endif -Requires: java-11-openjdk -Requires(pre): shadow-utils +Requires: %{name}-base = %{version}-%{release} +Requires: %{name}-base-libs = %{version}-%{release} +Requires: %{name}-clients = %{version}-%{release} +Requires: %{name}-config-model-fat = %{version}-%{release} +Requires: %{name}-jars = %{version}-%{release} +Requires: %{name}-malloc = %{version}-%{release} +Requires: %{name}-tools = %{version}-%{release} # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private # _dl_sym function. -Provides: libc.so.6(GLIBC_PRIVATE)(64bit) +%global __requires_exclude ^libc\\.so\\.6\\(GLIBC_PRIVATE\\)\\(64bit\\)$ + %description Vespa - The open big data serving engine +%package base + +Summary: Vespa - The open big data serving engine - base + +Requires: java-11-openjdk +Requires: perl +Requires: perl-Getopt-Long +Requires(pre): shadow-utils + +%description base + +Vespa - The open big data serving engine - base + +%package base-libs + +Summary: Vespa - The open big data serving engine - base C++ libs + +Requires: xxhash-libs >= 0.7.3 +Requires: lz4 +Requires: libzstd +%if 0%{?el7} +Requires: vespa-openssl >= 1.1.1c-1 +%else +Requires: openssl-libs +%endif + +%description base-libs + +Vespa - The open big data serving engine - base C++ libs + +%package clients + +Summary: Vespa - The open big data serving engine - clients + +%description clients + +Vespa - The open big data serving engine - clients + +%package config-model-fat + +Summary: Vespa - The open big data serving engine - config models + +%description config-model-fat + +Vespa - The open big data serving engine - config models + +%package node-admin + +Summary: Vespa - The open big data serving engine - node-admin + +Requires: %{name}-base = %{version}-%{release} +Requires: %{name}-jars = %{version}-%{release} + +%description node-admin + +Vespa - The open big data serving engine - node-admin + +%package jars + +Summary: Vespa - The open big data serving engine - shared java jar files + +%description jars + +Vespa - The open big data serving engine - shared java jar files + +%package malloc + +Summary: Vespa - The open big data serving engine - malloc library + +%description malloc + +Vespa - The open big data serving engine - malloc library + +%package tools + +Summary: Vespa - The open big data serving engine - tools + +Requires: %{name}-base = %{version}-%{release} +Requires: %{name}-base-libs = %{version}-%{release} + +%description tools + +Vespa - The open big data serving engine - tools + %prep %if 0%{?installdir:1} %setup -c -D -T @@ -245,7 +336,7 @@ cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DCMAKE_PREFIX_PATH=%{_vespa_deps_prefix} \ -DEXTRA_LINK_DIRECTORY="%{_extra_link_directory}" \ -DEXTRA_INCLUDE_DIRECTORY="%{_extra_include_directory}" \ - -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}};/usr/lib/jvm/jre-11-openjdk/lib" \ + -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}}" \ %{?_vespa_llvm_version:-DVESPA_LLVM_VERSION="%{_vespa_llvm_version}"} \ -DVESPA_USER=%{_vespa_user} \ -DVESPA_UNPRIVILEGED=no \ @@ -269,16 +360,18 @@ cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa.service %{buildroot}/usr/lib cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa-configserver.service %{buildroot}/usr/lib/systemd/system %endif +ln -s /usr/lib/jvm/jre-11-openjdk %{buildroot}/%{_prefix}/jdk + %clean rm -rf $RPM_BUILD_ROOT -%pre +%pre base %if %{_create_vespa_group} getent group %{_vespa_group} >/dev/null || groupadd -r %{_vespa_group} %endif %if %{_create_vespa_user} getent passwd %{_vespa_user} >/dev/null || \ - useradd -r -g %{_vespa_group} --home-dir %{_prefix} -s /sbin/nologin \ + useradd -r %{?_vespa_user_uid:-u %{_vespa_user_uid}} -g %{_vespa_group} --home-dir %{_prefix} -s /sbin/nologin \ -c "Create owner of all Vespa data files" %{_vespa_user} %endif echo "pathmunge %{_prefix}/bin" > /etc/profile.d/vespa.sh @@ -297,11 +390,13 @@ exit 0 %systemd_preun vespa-configserver.service %endif -%postun %if %{_create_vespa_service} +%postun %systemd_postun_with_restart vespa.service %systemd_postun_with_restart vespa-configserver.service %endif + +%postun base if [ $1 -eq 0 ]; then # this is an uninstallation rm -f /etc/profile.d/vespa.sh %if %{_create_vespa_user} @@ -311,6 +406,19 @@ if [ $1 -eq 0 ]; then # this is an uninstallation ! getent group %{_vespa_group} >/dev/null || groupdel %{_vespa_group} %endif fi +# Keep modifications to conf/vespa/default-env.txt across +# package uninstall + install. +if test -f %{_prefix}/conf/vespa/default-env.txt.rpmsave +then + if test -f %{_prefix}/conf/vespa/default-env.txt + then + # Temporarily remove default-env.txt.rpmsave when + # default-env.txt exists + rm -f %{_prefix}/conf/vespa/default-env.txt.rpmsave + else + mv %{_prefix}/conf/vespa/default-env.txt.rpmsave %{_prefix}/conf/vespa/default-env.txt + fi +fi %files %if %{_defattr_is_vespa_vespa} @@ -319,20 +427,73 @@ fi %doc %dir %{_prefix} %{_prefix}/bin +%exclude %{_prefix}/bin/vespa-destination +%exclude %{_prefix}/bin/vespa-document-statistics +%exclude %{_prefix}/bin/vespa-fbench +%exclude %{_prefix}/bin/vespa-feeder +%exclude %{_prefix}/bin/vespa-get +%exclude %{_prefix}/bin/vespa-logfmt +%exclude %{_prefix}/bin/vespa-query-profile-dump-tool +%exclude %{_prefix}/bin/vespa-stat +%exclude %{_prefix}/bin/vespa-security-env +%exclude %{_prefix}/bin/vespa-summary-benchmark +%exclude %{_prefix}/bin/vespa-visit +%exclude %{_prefix}/bin/vespa-visit-target %dir %{_prefix}/conf %{_prefix}/conf/configserver %{_prefix}/conf/configserver-app +%exclude %{_prefix}/conf/configserver-app/components/config-model-fat.jar +%exclude %{_prefix}/conf/configserver-app/config-models.xml %dir %{_prefix}/conf/logd -%{_prefix}/conf/node-admin-app %dir %{_prefix}/conf/vespa %dir %attr(-,%{_vespa_user},-) %{_prefix}/conf/zookeeper %dir %{_prefix}/etc %{_prefix}/etc/systemd %{_prefix}/etc/vespa +%exclude %{_prefix}/etc/vespamalloc.conf %{_prefix}/include -%{_prefix}/lib +%dir %{_prefix}/lib +%dir %{_prefix}/lib/jars +%{_prefix}/lib/jars/application-model-jar-with-dependencies.jar +%{_prefix}/lib/jars/application-preprocessor-jar-with-dependencies.jar +%{_prefix}/lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar +%{_prefix}/lib/jars/clustercontroller-apps-jar-with-dependencies.jar +%{_prefix}/lib/jars/clustercontroller-apputil-jar-with-dependencies.jar +%{_prefix}/lib/jars/clustercontroller-core-jar-with-dependencies.jar +%{_prefix}/lib/jars/clustercontroller-utils-jar-with-dependencies.jar +%{_prefix}/lib/jars/config-models +%{_prefix}/lib/jars/config-proxy-jar-with-dependencies.jar +%{_prefix}/lib/jars/configserver-flags-jar-with-dependencies.jar +%{_prefix}/lib/jars/configserver-jar-with-dependencies.jar +%{_prefix}/lib/jars/document.jar +%{_prefix}/lib/jars/filedistribution-jar-with-dependencies.jar +%{_prefix}/lib/jars/jdisc_jetty.jar +%{_prefix}/lib/jars/logserver-jar-with-dependencies.jar +%{_prefix}/lib/jars/metrics-proxy-jar-with-dependencies.jar +%{_prefix}/lib/jars/node-repository-jar-with-dependencies.jar +%{_prefix}/lib/jars/orchestrator-jar-with-dependencies.jar +%{_prefix}/lib/jars/predicate-search-jar-with-dependencies.jar +%{_prefix}/lib/jars/searchlib.jar +%{_prefix}/lib/jars/searchlib-jar-with-dependencies.jar +%{_prefix}/lib/jars/service-monitor-jar-with-dependencies.jar +%{_prefix}/lib/jars/vespa_feed_perf-jar-with-dependencies.jar +%{_prefix}/lib/jars/vespa-testrunner-components.jar +%{_prefix}/lib/jars/vespa-testrunner-components-jar-with-dependencies.jar +%{_prefix}/lib/jars/zookeeper-command-line-client-jar-with-dependencies.jar +%{_prefix}/lib/perl5 %{_prefix}/lib64 +%exclude %{_prefix}/lib64/libfastos.so +%exclude %{_prefix}/lib64/libfnet.so +%exclude %{_prefix}/lib64/libstaging_vespalib.so +%exclude %{_prefix}/lib64/libvespadefaults.so +%exclude %{_prefix}/lib64/libvespalib.so +%exclude %{_prefix}/lib64/libvespalog.so +%exclude %{_prefix}/lib64/vespa %{_prefix}/libexec +%exclude %{_prefix}/libexec/vespa/common-env.sh +%exclude %{_prefix}/libexec/vespa/node-admin.sh +%exclude %{_prefix}/libexec/vespa/standalone-container.sh +%exclude %{_prefix}/libexec/vespa/vespa-curl-wrapper %dir %attr(1777,-,-) %{_prefix}/logs %dir %attr(1777,%{_vespa_user},-) %{_prefix}/logs/vespa %dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/configserver @@ -349,11 +510,157 @@ fi %dir %attr(-,%{_vespa_user},-) %{_prefix}/var/db/vespa/logcontrol %dir %attr(-,%{_vespa_user},-) %{_prefix}/var/zookeeper %config(noreplace) %{_prefix}/conf/logd/logd.cfg -%config(noreplace) %{_prefix}/conf/vespa/default-env.txt -%config(noreplace) %{_prefix}/etc/vespamalloc.conf %if %{_create_vespa_service} %attr(644,root,root) /usr/lib/systemd/system/vespa.service %attr(644,root,root) /usr/lib/systemd/system/vespa-configserver.service %endif +%files base +%if %{_defattr_is_vespa_vespa} +%defattr(-,%{_vespa_user},%{_vespa_group},-) +%endif +%dir %{_prefix} +%dir %{_prefix}/bin +%{_prefix}/bin/vespa-logfmt +%{_prefix}/bin/vespa-security-env +%dir %{_prefix}/conf +%dir %{_prefix}/conf/vespa +%config(noreplace) %{_prefix}/conf/vespa/default-env.txt +%{_prefix}/jdk +%dir %{_prefix}/lib +%dir %{_prefix}/lib/jars +%{_prefix}/lib/jars/security-tools-jar-with-dependencies.jar +%dir %{_prefix}/libexec +%dir %{_prefix}/libexec/vespa +%{_prefix}/libexec/vespa/common-env.sh +%{_prefix}/libexec/vespa/vespa-curl-wrapper + +%files base-libs +%if %{_defattr_is_vespa_vespa} +%defattr(-,%{_vespa_user},%{_vespa_group},-) +%endif +%dir %{_prefix} +%dir %{_prefix}/lib64 +%{_prefix}/lib64/libfastos.so +%{_prefix}/lib64/libfnet.so +%{_prefix}/lib64/libstaging_vespalib.so +%{_prefix}/lib64/libvespadefaults.so +%{_prefix}/lib64/libvespalib.so +%{_prefix}/lib64/libvespalog.so + +%files clients +%if %{_defattr_is_vespa_vespa} +%defattr(-,%{_vespa_user},%{_vespa_group},-) +%endif +%dir %{_prefix} +%dir %{_prefix}/lib +%dir %{_prefix}/lib/jars +%{_prefix}/lib/jars/vespa-http-client-jar-with-dependencies.jar + +%files config-model-fat +%if %{_defattr_is_vespa_vespa} +%defattr(-,%{_vespa_user},%{_vespa_group},-) +%endif +%dir %{_prefix} +%dir %{_prefix}/conf +%dir %{_prefix}/conf/configserver-app +%dir %{_prefix}/conf/configserver-app/components +%{_prefix}/conf/configserver-app/components/config-model-fat.jar +%{_prefix}/conf/configserver-app/config-models.xml +%dir %{_prefix}/lib +%dir %{_prefix}/lib/jars +%{_prefix}/lib/jars/config-model-fat.jar + +%files node-admin +%if %{_defattr_is_vespa_vespa} +%defattr(-,%{_vespa_user},%{_vespa_group},-) +%endif +%dir %{_prefix} +%dir %{_prefix}/conf +%{_prefix}/conf/node-admin-app +%dir %{_prefix}/libexec +%dir %{_prefix}/libexec/vespa +%{_prefix}/libexec/vespa/node-admin.sh + +%files jars +%if %{_defattr_is_vespa_vespa} +%defattr(-,%{_vespa_user},%{_vespa_group},-) +%endif +%dir %{_prefix} +%dir %{_prefix}/lib +%dir %{_prefix}/lib/jars +%{_prefix}/lib/jars/aopalliance-repackaged-*.jar +%{_prefix}/lib/jars/bcpkix-jdk15on-*.jar +%{_prefix}/lib/jars/bcprov-jdk15on-*.jar +%{_prefix}/lib/jars/component-jar-with-dependencies.jar +%{_prefix}/lib/jars/config-bundle-jar-with-dependencies.jar +%{_prefix}/lib/jars/configdefinitions-jar-with-dependencies.jar +%{_prefix}/lib/jars/config-model-api-jar-with-dependencies.jar +%{_prefix}/lib/jars/config-model-jar-with-dependencies.jar +%{_prefix}/lib/jars/config-provisioning-jar-with-dependencies.jar +%{_prefix}/lib/jars/container-disc-jar-with-dependencies.jar +%{_prefix}/lib/jars/container-jersey2-jar-with-dependencies.jar +%{_prefix}/lib/jars/container-search-and-docproc-jar-with-dependencies.jar +%{_prefix}/lib/jars/container-search-gui-jar-with-dependencies.jar +%{_prefix}/lib/jars/defaults-jar-with-dependencies.jar +%{_prefix}/lib/jars/docprocs-jar-with-dependencies.jar +%{_prefix}/lib/jars/flags-jar-with-dependencies.jar +%{_prefix}/lib/jars/hk2-*.jar +%{_prefix}/lib/jars/jackson-*.jar +%{_prefix}/lib/jars/javassist-*.jar +%{_prefix}/lib/jars/javax.*.jar +%{_prefix}/lib/jars/jdisc_core-jar-with-dependencies.jar +%{_prefix}/lib/jars/jdisc_http_service-jar-with-dependencies.jar +%{_prefix}/lib/jars/jdisc-security-filters-jar-with-dependencies.jar +%{_prefix}/lib/jars/jersey-*.jar +%{_prefix}/lib/jars/jetty-*.jar +%{_prefix}/lib/jars/mimepull-*.jar +%{_prefix}/lib/jars/model-evaluation-jar-with-dependencies.jar +%{_prefix}/lib/jars/model-integration-jar-with-dependencies.jar +%{_prefix}/lib/jars/osgi-resource-locator-*.jar +%{_prefix}/lib/jars/security-utils-jar-with-dependencies.jar +%{_prefix}/lib/jars/simplemetrics-jar-with-dependencies.jar +%{_prefix}/lib/jars/standalone-container-jar-with-dependencies.jar +%{_prefix}/lib/jars/validation-api-*.jar +%{_prefix}/lib/jars/vespa-athenz-jar-with-dependencies.jar +%{_prefix}/lib/jars/vespaclient-container-plugin-jar-with-dependencies.jar +%{_prefix}/lib/jars/vespajlib.jar +%{_prefix}/lib/jars/zkfacade-jar-with-dependencies.jar +%{_prefix}/lib/jars/zookeeper-server-*-jar-with-dependencies.jar +%{_prefix}/lib/jars/zookeeper-server-common-jar-with-dependencies.jar +%{_prefix}/lib/jars/zookeeper-server-jar-with-dependencies.jar +%dir %{_prefix}/libexec +%dir %{_prefix}/libexec/vespa +%{_prefix}/libexec/vespa/standalone-container.sh + +%files malloc +%if %{_defattr_is_vespa_vespa} +%defattr(-,%{_vespa_user},%{_vespa_group},-) +%endif +%dir %{_prefix} +%dir %{_prefix}/etc +%config(noreplace) %{_prefix}/etc/vespamalloc.conf +%dir %{_prefix}/lib64 +%{_prefix}/lib64/vespa + +%files tools +%if %{_defattr_is_vespa_vespa} +%defattr(-,%{_vespa_user},%{_vespa_group},-) +%endif +%dir %{_prefix} +%dir %{_prefix}/bin +%{_prefix}/bin/vespa-destination +%{_prefix}/bin/vespa-document-statistics +%{_prefix}/bin/vespa-fbench +%{_prefix}/bin/vespa-feeder +%{_prefix}/bin/vespa-get +%{_prefix}/bin/vespa-query-profile-dump-tool +%{_prefix}/bin/vespa-stat +%{_prefix}/bin/vespa-summary-benchmark +%{_prefix}/bin/vespa-visit +%{_prefix}/bin/vespa-visit-target +%dir %{_prefix}/lib +%dir %{_prefix}/lib/jars +%{_prefix}/lib/jars/vespaclient-java-jar-with-dependencies.jar + %changelog diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 110153954af..9ac402f56ef 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -18,6 +18,7 @@ #include <vespa/document/select/compare.h> #include <vespa/document/select/operator.h> #include <vespa/document/select/parse_utils.h> +#include <vespa/document/select/parser_limits.h> #include <vespa/vespalib/util/exceptions.h> #include <limits> #include <gtest/gtest.h> @@ -33,6 +34,8 @@ protected: std::vector<Document::SP > _doc; std::vector<DocumentUpdate::SP > _update; + ~DocumentSelectParserTest(); + Document::SP createDoc( const std::string& doctype, const std::string& id, uint32_t hint, double hfloat, const std::string& hstr, const std::string& cstr, @@ -64,6 +67,7 @@ protected: void testDocumentUpdates4(); }; +DocumentSelectParserTest::~DocumentSelectParserTest() = default; namespace { std::shared_ptr<const DocumentTypeRepo> _repo; @@ -1247,17 +1251,17 @@ TEST_F(DocumentSelectParserTest, testThatSimpleFieldValuesHaveCorrectFieldName) TEST_F(DocumentSelectParserTest, testThatComplexFieldValuesHaveCorrectFieldNames) { - EXPECT_EQ( - vespalib::string("headerval"), - parseFieldValue("testdoctype1.headerval{test}")->getRealFieldName()); + EXPECT_EQ(vespalib::string("headerval"), + parseFieldValue("testdoctype1.headerval{test}")->getRealFieldName()); - EXPECT_EQ( - vespalib::string("headerval"), - parseFieldValue("testdoctype1.headerval[42]")->getRealFieldName()); + EXPECT_EQ(vespalib::string("headerval"), + parseFieldValue("testdoctype1.headerval[42]")->getRealFieldName()); - EXPECT_EQ( - vespalib::string("headerval"), - parseFieldValue("testdoctype1.headerval.meow.meow{test}")->getRealFieldName()); + EXPECT_EQ(vespalib::string("headerval"), + parseFieldValue("testdoctype1.headerval.meow.meow{test}")->getRealFieldName()); + + EXPECT_EQ(vespalib::string("headerval"), + parseFieldValue("testdoctype1.headerval .meow.meow{test}")->getRealFieldName()); } namespace { @@ -1603,4 +1607,64 @@ TEST_F(DocumentSelectParserTest, redundant_glob_wildcards_are_collapsed_into_min EXPECT_EQ(GlobOperator::convertToRegex("*?*?*?*"), "..*..*."); // Don't try this at home, kids! } +TEST_F(DocumentSelectParserTest, recursion_depth_is_bounded_for_field_exprs) { + createDocs(); + std::string expr = "testdoctype1"; + for (size_t i = 0; i < 50000; ++i) { + expr += ".foo"; + } + expr += ".hash() != 0"; + verifyFailedParse(expr, "ParsingFailedException: expression is too deeply nested (max 1024 levels)"); +} + +TEST_F(DocumentSelectParserTest, recursion_depth_is_bounded_for_arithmetic_exprs) { + createDocs(); + std::string expr = "1"; + for (size_t i = 0; i < 50000; ++i) { + expr += "+1"; + } + expr += " != 0"; + verifyFailedParse(expr, "ParsingFailedException: expression is too deeply nested (max 1024 levels)"); +} + +TEST_F(DocumentSelectParserTest, recursion_depth_is_bounded_for_binary_logical_exprs) { + createDocs(); + // Also throw in some comparisons to ensure they carry over the max depth. + std::string expr = "1 == 2"; + std::string cmp_subexpr = "3 != 4"; + for (size_t i = 0; i < 10000; ++i) { + expr += (i % 2 == 0 ? " and " : " or ") + cmp_subexpr; + } + verifyFailedParse(expr, "ParsingFailedException: expression is too deeply nested (max 1024 levels)"); +} + +TEST_F(DocumentSelectParserTest, recursion_depth_is_bounded_for_unary_logical_exprs) { + createDocs(); + std::string expr; + for (size_t i = 0; i < 10000; ++i) { + expr += "not "; + } + expr += "true"; + verifyFailedParse(expr, "ParsingFailedException: expression is too deeply nested (max 1024 levels)"); +} + +TEST_F(DocumentSelectParserTest, selection_has_upper_limit_on_input_size) { + createDocs(); + std::string expr = ("testdoctype1.a_biii" + + std::string(select::ParserLimits::MaxSelectionByteSize, 'i') + + "iiig_identifier"); + verifyFailedParse(expr, "ParsingFailedException: expression is too large to be " + "parsed (max 1048576 bytes, got 1048610)"); +} + +TEST_F(DocumentSelectParserTest, lexing_does_not_have_superlinear_time_complexity) { + createDocs(); + std::string expr = ("testdoctype1.hstringval == 'a_biii" + + std::string(select::ParserLimits::MaxSelectionByteSize - 100, 'i') + + "iiig string'"); + // If the lexer is not compiled with the appropriate options, this will take a long time. + // A really, really long time. + PARSE(expr, *_doc[0], False); +} + } // document diff --git a/document/src/vespa/document/select/CMakeLists.txt b/document/src/vespa/document/select/CMakeLists.txt index 81e5d86675c..f210e8abdd7 100644 --- a/document/src/vespa/document/select/CMakeLists.txt +++ b/document/src/vespa/document/select/CMakeLists.txt @@ -36,6 +36,7 @@ vespa_add_library(document_select OBJECT parser.cpp parse_utils.cpp parsing_failed_exception.cpp + parser_limits.cpp ${BISON_DocSelParser_OUTPUTS} ${FLEX_DocSelLexer_OUTPUTS} AFTER diff --git a/document/src/vespa/document/select/branch.cpp b/document/src/vespa/document/select/branch.cpp index b3d5f97ccab..9104e2c5544 100644 --- a/document/src/vespa/document/select/branch.cpp +++ b/document/src/vespa/document/select/branch.cpp @@ -8,7 +8,7 @@ namespace document::select { And::And(std::unique_ptr<Node> left, std::unique_ptr<Node> right, const char* name) - : Branch(name ? name : "and"), + : Branch(name ? name : "and", std::max(left->max_depth(), right->max_depth()) + 1), _left(std::move(left)), _right(std::move(right)) { @@ -54,7 +54,7 @@ And::trace(const Context& context, std::ostream& out) const } Or::Or(std::unique_ptr<Node> left, std::unique_ptr<Node> right, const char* name) - : Branch(name ? name : "or"), + : Branch(name ? name : "or", std::max(left->max_depth(), right->max_depth()) + 1), _left(std::move(left)), _right(std::move(right)) { @@ -100,7 +100,7 @@ Or::trace(const Context& context, std::ostream& out) const } Not::Not(std::unique_ptr<Node> child, const char* name) - : Branch(name ? name : "not"), + : Branch(name ? name : "not", child->max_depth() + 1), _child(std::move(child)) { assert(_child.get()); diff --git a/document/src/vespa/document/select/branch.h b/document/src/vespa/document/select/branch.h index 8637b41de89..77ed74030b5 100644 --- a/document/src/vespa/document/select/branch.h +++ b/document/src/vespa/document/select/branch.h @@ -19,7 +19,8 @@ namespace document::select { class Branch : public Node { public: - Branch(vespalib::stringref name) : Node(name) {} + explicit Branch(vespalib::stringref name) : Node(name) {} + Branch(vespalib::stringref name, uint32_t max_depth) : Node(name, max_depth) {} bool isLeafNode() const override { return false; } }; @@ -30,7 +31,7 @@ class And : public Branch std::unique_ptr<Node> _right; public: And(std::unique_ptr<Node> left, std::unique_ptr<Node> right, - const char* name = 0); + const char* name = nullptr); ResultList contains(const Context& context) const override { return (_left->contains(context) && _right->contains(context)); @@ -53,7 +54,7 @@ class Or : public Branch std::unique_ptr<Node> _right; public: Or(std::unique_ptr<Node> left, std::unique_ptr<Node> right, - const char* name = 0); + const char* name = nullptr); ResultList contains(const Context& context) const override { return (_left->contains(context) || _right->contains(context)); @@ -74,7 +75,7 @@ class Not : public Branch { std::unique_ptr<Node> _child; public: - Not(std::unique_ptr<Node> child, const char* name = 0); + Not(std::unique_ptr<Node> child, const char* name = nullptr); ResultList contains(const Context& context) const override { return !_child->contains(context); } ResultList trace(const Context&, std::ostream& trace) const override; diff --git a/document/src/vespa/document/select/compare.cpp b/document/src/vespa/document/select/compare.cpp index 7db40929a64..caef1bdd250 100644 --- a/document/src/vespa/document/select/compare.cpp +++ b/document/src/vespa/document/select/compare.cpp @@ -15,7 +15,7 @@ Compare::Compare(std::unique_ptr<ValueNode> left, const Operator& op, std::unique_ptr<ValueNode> right, const BucketIdFactory& bucketIdFactory) - : Node("Compare"), + : Node("Compare", std::max(left->max_depth(), right->max_depth()) + 1), _left(std::move(left)), _right(std::move(right)), _operator(op), diff --git a/document/src/vespa/document/select/grammar/lexer.ll b/document/src/vespa/document/select/grammar/lexer.ll index bd011c8ebf6..1222aac02a2 100644 --- a/document/src/vespa/document/select/grammar/lexer.ll +++ b/document/src/vespa/document/select/grammar/lexer.ll @@ -7,6 +7,13 @@ %option noyywrap nounput %option yyclass="document::select::DocSelScanner" + /* Flex lexer must be compiled with batch mode (as opposed to interactive mode) + * or parsing of large tokens appears to trigger superlinear time complexity. + * Also use full, non-compressed lookup tables for maximum performance. + */ +%option batch +%option full + /* Used to track source locations, see https://github.com/bingmann/flex-bison-cpp-example/blob/master/src/scanner.ll */ %{ #define YY_USER_ACTION yyloc->columns(yyleng); diff --git a/document/src/vespa/document/select/node.h b/document/src/vespa/document/select/node.h index 48a64ae63f5..9a3b687d81c 100644 --- a/document/src/vespa/document/select/node.h +++ b/document/src/vespa/document/select/node.h @@ -12,6 +12,7 @@ #include "resultlist.h" #include "context.h" +#include "parser_limits.h" namespace document::select { @@ -21,19 +22,33 @@ class Node : public Printable { protected: vespalib::string _name; + uint32_t _max_depth; bool _parentheses; // Set to true if parentheses was used around this part // Set such that we can recreate original query in print. public: typedef std::unique_ptr<Node> UP; typedef std::shared_ptr<Node> SP; - Node(vespalib::stringref name) : _name(name), _parentheses(false) {} - ~Node() override {} + Node(vespalib::stringref name, uint32_t max_depth) + : _name(name), _max_depth(max_depth), _parentheses(false) + { + throw_parse_error_if_max_depth_exceeded(); + } - void setParentheses() { _parentheses = true; } + explicit Node(vespalib::stringref name) + : _name(name), _max_depth(1), _parentheses(false) + {} + ~Node() override = default; - void clearParentheses() { _parentheses = false; } + // Depth is explicitly tracked to limit recursion to a sane maximum when building and + // processing ASTs, as the Bison framework does not have anything useful for us there. + // The AST is built from the leaves up towards the root, so we can cheaply track depth + // of subtrees in O(1) time per node by computing a node's own depth based on immediate + // children at node construction time. + [[nodiscard]] uint32_t max_depth() const noexcept { return _max_depth; } + void setParentheses() { _parentheses = true; } + void clearParentheses() { _parentheses = false; } bool hadParentheses() const { return _parentheses; } virtual ResultList contains(const Context&) const = 0; @@ -43,6 +58,12 @@ public: virtual Node::UP clone() const = 0; protected: + void throw_parse_error_if_max_depth_exceeded() const { + if (_max_depth > ParserLimits::MaxRecursionDepth) { + throw_max_depth_exceeded_exception(); + } + } + Node::UP wrapParens(Node* node) const { Node::UP ret(node); if (_parentheses) { diff --git a/document/src/vespa/document/select/parser.cpp b/document/src/vespa/document/select/parser.cpp index 9f015409011..fadb46e5aa3 100644 --- a/document/src/vespa/document/select/parser.cpp +++ b/document/src/vespa/document/select/parser.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "parser.h" +#include "parser_limits.h" #include "scanner.h" #include <vespa/document/base/exceptions.h> #include <vespa/document/util/stringutil.h> @@ -8,7 +9,20 @@ namespace document::select { +namespace { + +void verify_expression_not_too_large(const std::string& expr) { + if (expr.size() > ParserLimits::MaxSelectionByteSize) { + throw ParsingFailedException(vespalib::make_string( + "expression is too large to be parsed (max %zu bytes, got %zu)", + ParserLimits::MaxSelectionByteSize, expr.size())); + } +} + +} + std::unique_ptr<Node> Parser::parse(const std::string& str) const { + verify_expression_not_too_large(str); try { std::istringstream ss(str); DocSelScanner scanner(&ss); diff --git a/document/src/vespa/document/select/parser_limits.cpp b/document/src/vespa/document/select/parser_limits.cpp new file mode 100644 index 00000000000..13e494b376f --- /dev/null +++ b/document/src/vespa/document/select/parser_limits.cpp @@ -0,0 +1,13 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "parser_limits.h" +#include "parsing_failed_exception.h" +#include <vespa/vespalib/util/stringfmt.h> + +namespace document::select { + +void throw_max_depth_exceeded_exception() { + throw ParsingFailedException(vespalib::make_string( + "expression is too deeply nested (max %u levels)", ParserLimits::MaxRecursionDepth)); +} + +} diff --git a/document/src/vespa/document/select/parser_limits.h b/document/src/vespa/document/select/parser_limits.h new file mode 100644 index 00000000000..24c0a165611 --- /dev/null +++ b/document/src/vespa/document/select/parser_limits.h @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <cstdint> +#include <cstddef> + +namespace document::select { + +// Any resource constraints set for parsing document selection expressions +struct ParserLimits { + // Max depth allowed for nodes in the AST tree. + constexpr static uint32_t MaxRecursionDepth = 1024; + // Max size of entire input document selection string, in bytes. + constexpr static size_t MaxSelectionByteSize = 1024*1024; +}; + +void __attribute__((noinline)) throw_max_depth_exceeded_exception(); + +} diff --git a/document/src/vespa/document/select/valuenode.h b/document/src/vespa/document/select/valuenode.h index 04ed8178b40..8dd535a736a 100644 --- a/document/src/vespa/document/select/valuenode.h +++ b/document/src/vespa/document/select/valuenode.h @@ -5,12 +5,13 @@ * * @brief Node representing a value in the tree * - * @author H�kon Humberset + * @author HÃ¥kon Humberset */ #pragma once #include "value.h" +#include "parser_limits.h" namespace document::select { @@ -22,8 +23,19 @@ class ValueNode : public Printable public: using UP = std::unique_ptr<ValueNode>; - ValueNode() : _parentheses(false) {} - virtual ~ValueNode() {} + explicit ValueNode(uint32_t max_depth) + : _max_depth(max_depth), _parentheses(false) + { + throw_parse_error_if_max_depth_exceeded(); + } + ValueNode() : _max_depth(1), _parentheses(false) {} + ~ValueNode() override = default; + + // See comments for same function in node.h for a description on how and why + // we track this. Since Node and ValueNode live in completely separate type + // hierarchies, this particular bit of code duplication is unfortunate but + // incurs the least cognitive overhead. + [[nodiscard]] uint32_t max_depth() const noexcept { return _max_depth; } void setParentheses() { _parentheses = true; } void clearParentheses() { _parentheses = false; } @@ -34,9 +46,17 @@ public: virtual ValueNode::UP clone() const = 0; virtual std::unique_ptr<Value> traceValue(const Context &context, std::ostream &out) const; private: + uint32_t _max_depth; bool _parentheses; // Set to true if parentheses was used around this part // Set such that we can recreate original query in print. + protected: + void throw_parse_error_if_max_depth_exceeded() const { + if (_max_depth > ParserLimits::MaxRecursionDepth) { + throw_max_depth_exceeded_exception(); + } + } + ValueNode::UP wrapParens(ValueNode* node) const { ValueNode::UP ret(node); if (_parentheses) { diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 95cf2f4e7e5..026623cf83c 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -21,10 +21,6 @@ LOG_SETUP(".document.select.valuenode"); namespace document::select { namespace { - static const std::regex FIELD_NAME_REGEX("^([_A-Za-z][_A-Za-z0-9]*).*"); -} - -namespace { bool documentTypeEqualsName(const DocumentType& type, vespalib::stringref name) { if (type.getName() == name) return true; @@ -40,7 +36,7 @@ namespace { InvalidValueNode::InvalidValueNode(vespalib::stringref name) : _name(name) -{ } +{} void @@ -194,15 +190,33 @@ FieldValueNode::FieldValueNode(const vespalib::string& doctype, FieldValueNode::~FieldValueNode() = default; -vespalib::string -FieldValueNode::extractFieldName(const std::string & fieldExpression) { - std::smatch match; +namespace { - if (std::regex_match(fieldExpression, match, FIELD_NAME_REGEX) && match[1].matched) { - return vespalib::string(match[1].first, match[1].second); +size_t first_ident_length_or_npos(const vespalib::string& expr) { + for (size_t i = 0; i < expr.size(); ++i) { + switch (expr[i]) { + case '.': + case '{': + case '[': + case ' ': + case '\n': + case '\t': + return i; + default: + continue; + } } + return vespalib::string::npos; +} - throw ParsingFailedException("Fatal: could not extract field name from field expression '" + fieldExpression + "'"); +} + +// TODO remove this pile of fun in favor of actually parsed AST nodes...! +vespalib::string +FieldValueNode::extractFieldName(const vespalib::string & fieldExpression) { + // When we get here the actual contents of the field expression shall already + // have been structurally and syntactically verified by the parser. + return fieldExpression.substr(0, first_ident_length_or_npos(fieldExpression)); } namespace { @@ -844,7 +858,8 @@ FunctionValueNode::print(std::ostream& out, bool verbose, ArithmeticValueNode::ArithmeticValueNode( std::unique_ptr<ValueNode> left, vespalib::stringref op, std::unique_ptr<ValueNode> right) - : _operator(), + : ValueNode(std::max(left->max_depth(), right->max_depth()) + 1), + _operator(), _left(std::move(left)), _right(std::move(right)) { diff --git a/document/src/vespa/document/select/valuenodes.h b/document/src/vespa/document/select/valuenodes.h index 8009542c364..a7d5fa15f37 100644 --- a/document/src/vespa/document/select/valuenodes.h +++ b/document/src/vespa/document/select/valuenodes.h @@ -160,7 +160,7 @@ public: FieldValueNode & operator = (const FieldValueNode &) = delete; FieldValueNode(FieldValueNode &&) = default; FieldValueNode & operator = (FieldValueNode &&) = default; - ~FieldValueNode(); + ~FieldValueNode() override; const vespalib::string& getDocType() const { return _doctype; } const vespalib::string& getRealFieldName() const { return _fieldName; } @@ -175,7 +175,7 @@ public: return wrapParens(new FieldValueNode(_doctype, _fieldExpression)); } - static vespalib::string extractFieldName(const std::string & fieldExpression); + static vespalib::string extractFieldName(const vespalib::string & fieldExpression); private: @@ -192,13 +192,15 @@ class FieldExprNode final : public ValueNode { public: explicit FieldExprNode(const vespalib::string& doctype) : _left_expr(), _right_expr(doctype) {} FieldExprNode(std::unique_ptr<FieldExprNode> left_expr, vespalib::stringref right_expr) - : _left_expr(std::move(left_expr)), _right_expr(right_expr) + : ValueNode(left_expr->max_depth() + 1), + _left_expr(std::move(left_expr)), + _right_expr(right_expr) {} FieldExprNode(const FieldExprNode &) = delete; FieldExprNode & operator = (const FieldExprNode &) = delete; FieldExprNode(FieldExprNode &&) = default; FieldExprNode & operator = (FieldExprNode &&) = default; - ~FieldExprNode(); + ~FieldExprNode() override; std::unique_ptr<FieldValueNode> convert_to_field_value() const; std::unique_ptr<FunctionValueNode> convert_to_function_call() const; diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-CreateVisitorMessage.dat Binary files differindex a7bb5b0e896..27e64170701 100644 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-CreateVisitorMessage.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-CreateVisitorMessage.dat diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 66b86866c3e..8a0212211e1 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -37,7 +37,8 @@ public class FileReferenceDownloader { private final static Duration rpcTimeout = Duration.ofSeconds(10); private final ExecutorService downloadExecutor = - Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), new DaemonThreadFactory("filereference downloader")); + Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), + new DaemonThreadFactory("filereference downloader")); private final ConnectionPool connectionPool; private final Map<FileReference, FileReferenceDownload> downloads = new LinkedHashMap<>(); private final Map<FileReference, Double> downloadStatus = new HashMap<>(); // between 0 and 1 diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index c3d9d74ed68..862d83461fd 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -67,6 +67,13 @@ public class Flags { "Takes effect on next host admin tick.", HOSTNAME); + public static final UnboundBooleanFlag USE_NEW_VESPA_RPMS = defineFeatureFlag( + "use-new-vespa-rpms", false, + "Whether to use the new vespa-rpms YUM repo when upgrading/downgrading. The vespa-version " + + "when fetching the flag value is the wanted version of the host.", + "Takes effect when upgrading or downgrading host admin to a different version.", + HOSTNAME, NODE_TYPE, VESPA_VERSION); + public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag( "disabled-host-admin-tasks", List.of(), String.class, "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask) that should be skipped", @@ -103,7 +110,7 @@ public class Flags { APPLICATION_ID); public static final UnboundStringFlag TLS_INSECURE_MIXED_MODE = defineStringFlag( - "tls-insecure-mixed-mode", "tls_client_mixed_server", + "tls-insecure-mixed-mode", "tls_client_tls_server", "TLS insecure mixed mode. Allowed values: ['plaintext_client_mixed_server', 'tls_client_mixed_server', 'tls_client_tls_server']", "Takes effect on restart of Docker container", NODE_TYPE, APPLICATION_ID, HOSTNAME); @@ -193,12 +200,6 @@ public class Flags { "Takes effect on next node agent tick (but does not clear existing failure reports)", HOSTNAME); - public static final UnboundBooleanFlag GENERATE_L4_ROUTING_CONFIG = defineFeatureFlag( - "generate-l4-routing-config", false, - "Whether routing nodes should generate L4 routing config", - "Takes effect immediately", - ZONE_ID, HOSTNAME); - public static final UnboundBooleanFlag USE_REFRESHED_ENDPOINT_CERTIFICATE = defineFeatureFlag( "use-refreshed-endpoint-certificate", false, "Whether an application should start using a newer certificate/key pair if available", @@ -251,17 +252,22 @@ public class Flags { APPLICATION_ID); public static final UnboundBooleanFlag DISABLE_ROUTING_GENERATOR = defineFeatureFlag( - "disable-routing-generator", false, + "disable-routing-generator", true, "Whether the controller should stop asking the routing layer for endpoints", "Takes effect immediately", APPLICATION_ID); public static final UnboundBooleanFlag DEDICATED_NODES_WHEN_UNSPECIFIED = defineFeatureFlag( - "dedicated-nodes-when-unspecified", false, + "dedicated-nodes-when-unspecified", true, "Whether config-server should allocate dedicated container nodes when <nodes/> is not specified in services.xml", "Takes effect on redeploy", APPLICATION_ID); + public static final UnboundBooleanFlag NGINX_UPSTREAM_PROXY_PROTOCOL = defineFeatureFlag( + "nginx-upstream-proxy-protocol", false, + "Whether the nginx should enable proxy-protocol for all upstreams", + "Takes effect immediately"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java index afaf94b26b6..db2f0a3a197 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java @@ -5,7 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.json.wire.WireCondition; -import java.util.List; import java.util.function.Predicate; /** diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalPredicate.java b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalPredicate.java index c5ad195e0d2..72bc5627112 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalPredicate.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalPredicate.java @@ -23,7 +23,7 @@ public class RelationalPredicate { for (var operator : operatorsByDecendingLength) { if (predicateString.startsWith(operator.toText())) { - String suffix = predicateString.substring(operator.toText().length()); + String suffix = predicateString.substring(operator.toText().length()).trim(); return new RelationalPredicate(predicateString, operator, suffix); } } diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index 84d18350993..ee30f6fd471 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -434,15 +434,16 @@ public abstract class ControllerHttpClient { private static DeploymentLog.Status valueOf(String status) { switch (status) { - case "running": return DeploymentLog.Status.running; - case "aborted": return DeploymentLog.Status.aborted; - case "error": return DeploymentLog.Status.error; - case "testFailure": return DeploymentLog.Status.testFailure; - case "outOfCapacity": return DeploymentLog.Status.outOfCapacity; - case "installationFailed": return DeploymentLog.Status.installationFailed; - case "deploymentFailed": return DeploymentLog.Status.deploymentFailed; - case "success": return DeploymentLog.Status.success; - default: throw new IllegalArgumentException("Unexpected status '" + status + "'"); + case "running": return DeploymentLog.Status.running; + case "aborted": return DeploymentLog.Status.aborted; + case "error": return DeploymentLog.Status.error; + case "testFailure": return DeploymentLog.Status.testFailure; + case "outOfCapacity": return DeploymentLog.Status.outOfCapacity; + case "installationFailed": return DeploymentLog.Status.installationFailed; + case "deploymentFailed": return DeploymentLog.Status.deploymentFailed; + case "endpointCertificateTimeout": return DeploymentLog.Status.endpointCertificateTimeout; + case "success": return DeploymentLog.Status.success; + default: throw new IllegalArgumentException("Unexpected status '" + status + "'"); } } diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java index 9eae9a33cff..90e973da49d 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java @@ -112,6 +112,7 @@ public class DeploymentLog { outOfCapacity, installationFailed, deploymentFailed, + endpointCertificateTimeout, success; } diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java index fd640bcc235..22c32cfa9ec 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java @@ -52,7 +52,7 @@ public class Properties { return getNonBlankProperty("apiCertificateFile").map(Paths::get); } - /** Returns the actual private key as a string */ + /** Returns the actual private key as a string. */ public static Optional<String> apiKey() { return getNonBlankProperty("apiKey"); } diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index bb6285ab94f..c5a0a676a70 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -42,6 +42,9 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Builder tlsClientAuthEnforcer(com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder healthCheckProxy(com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder proxyProtocol(com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol$Builder)", + "public com.yahoo.jdisc.http.ConnectorConfig$Builder secureRedirect(com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder)", + "public com.yahoo.jdisc.http.ConnectorConfig$Builder maxRequestsPerConnection(int)", + "public com.yahoo.jdisc.http.ConnectorConfig$Builder maxConnectionLife(double)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", "public final java.lang.String getDefMd5()", "public final java.lang.String getDefName()", @@ -53,7 +56,8 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder ssl", "public com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder tlsClientAuthEnforcer", "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder healthCheckProxy", - "public com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol$Builder proxyProtocol" + "public com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol$Builder proxyProtocol", + "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder secureRedirect" ] }, "com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder": { @@ -133,6 +137,37 @@ ], "fields": [] }, + "com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect)", + "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder enabled(boolean)", + "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder port(int)", + "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect build()" + ], + "fields": [] + }, + "com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder)", + "public boolean enabled()", + "public int port()" + ], + "fields": [] + }, "com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder": { "superClass": "java.lang.Object", "interfaces": [ @@ -323,7 +358,10 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl ssl()", "public com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer tlsClientAuthEnforcer()", "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy healthCheckProxy()", - "public com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol proxyProtocol()" + "public com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol proxyProtocol()", + "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect secureRedirect()", + "public int maxRequestsPerConnection()", + "public double maxConnectionLife()" ], "fields": [ "public static final java.lang.String CONFIG_DEF_MD5", diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index 71dcb7d0682..b9d686c1d6b 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -10,6 +10,7 @@ import com.yahoo.jdisc.handler.BindingNotFoundException; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.OverloadException; import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.jdisc.http.HttpRequest; import org.eclipse.jetty.io.EofException; @@ -22,6 +23,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.Arrays; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -34,6 +36,7 @@ import java.util.logging.Logger; import static com.yahoo.jdisc.http.HttpHeaders.Values.APPLICATION_X_WWW_FORM_URLENCODED; import static com.yahoo.jdisc.http.core.HttpServletRequestUtils.getConnection; import static com.yahoo.jdisc.http.server.jetty.Exceptions.throwUnchecked; +import static com.yahoo.jdisc.http.server.jetty.JDiscHttpServlet.getConnector; /** * @author Simon Thoresen Hult @@ -64,14 +67,13 @@ class HttpRequestDispatch { this.jettyRequest = (Request) servletRequest; this.metricReporter = new MetricReporter(jDiscContext.metric, metricContext, jettyRequest.getTimeStamp()); - honourMaxKeepAliveRequests(); this.servletResponseController = new ServletResponseController( servletRequest, servletResponse, jDiscContext.janitor, metricReporter, jDiscContext.developerMode()); - + markConnectionAsNonPersistentIfThresholdReached(servletRequest); this.async = servletRequest.startAsync(); async.setTimeout(0); metricReporter.uriLength(jettyRequest.getOriginalURI().length()); @@ -102,15 +104,6 @@ class HttpRequestDispatch { } } - private void honourMaxKeepAliveRequests() { - if (jDiscContext.serverConfig.maxKeepAliveRequests() > 0) { - HttpConnection connection = getConnection(jettyRequest); - if (connection.getMessagesIn() >= jDiscContext.serverConfig.maxKeepAliveRequests()) { - connection.getGenerator().setPersistent(false); - } - } - } - private BiConsumer<Void, Throwable> completeRequestCallback; { AtomicBoolean completeRequestCalled = new AtomicBoolean(false); @@ -151,6 +144,25 @@ class HttpRequestDispatch { }; } + private static void markConnectionAsNonPersistentIfThresholdReached(HttpServletRequest request) { + ConnectorConfig connectorConfig = getConnector(request).connectorConfig(); + int maxRequestsPerConnection = connectorConfig.maxRequestsPerConnection(); + if (maxRequestsPerConnection > 0) { + HttpConnection connection = getConnection(request); + if (connection.getMessagesIn() >= maxRequestsPerConnection) { + connection.getGenerator().setPersistent(false); + } + } + double maxConnectionLifeInSeconds = connectorConfig.maxConnectionLife(); + if (maxConnectionLifeInSeconds > 0) { + HttpConnection connection = getConnection(request); + Instant expireAt = Instant.ofEpochMilli((long)(connection.getCreatedTimeStamp() + maxConnectionLifeInSeconds * 1000)); + if (Instant.now().isAfter(expireAt)) { + connection.getGenerator().setPersistent(false); + } + } + } + @SafeVarargs @SuppressWarnings("varargs") private static boolean isErrorOfType(Throwable throwable, Class<? extends Throwable>... handledTypes) { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java index cf66af31a79..5cbe7320f0e 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java @@ -100,6 +100,8 @@ class JDiscHttpServlet extends HttpServlet { } } + + static JDiscServerConnector getConnector(HttpServletRequest request) { return (JDiscServerConnector)getConnection(request).getConnector(); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index 71284e09669..c5f42ff9cc5 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -34,9 +34,6 @@ import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.log.JavaUtilLog; import org.eclipse.jetty.util.log.Log; import org.eclipse.jetty.util.thread.QueuedThreadPool; -import org.osgi.framework.BundleContext; -import org.osgi.framework.InvalidSyntaxException; -import org.osgi.framework.ServiceReference; import javax.management.remote.JMXServiceURL; import javax.servlet.DispatcherType; @@ -44,10 +41,8 @@ import java.io.IOException; import java.lang.management.ManagementFactory; import java.net.BindException; import java.net.MalformedURLException; -import java.nio.channels.ServerSocketChannel; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -246,10 +241,13 @@ public class JettyHttpServer extends AbstractServerProvider { servletContextHandler.addServlet(jdiscServlet, "/*"); + List<ConnectorConfig> connectorConfigs = connectors.stream().map(JDiscServerConnector::connectorConfig).collect(toList()); + var secureRedirectHandler = new SecuredRedirectHandler(connectorConfigs); + secureRedirectHandler.setHandler(servletContextHandler); + var proxyHandler = new HealthCheckProxyHandler(connectors); - proxyHandler.setHandler(servletContextHandler); + proxyHandler.setHandler(secureRedirectHandler); - List<ConnectorConfig> connectorConfigs = connectors.stream().map(JDiscServerConnector::connectorConfig).collect(toList()); var authEnforcer = new TlsClientAuthenticationEnforcer(connectorConfigs); authEnforcer.setHandler(proxyHandler); @@ -282,25 +280,6 @@ public class JettyHttpServer extends AbstractServerProvider { return ports.stream().map(Object::toString).collect(Collectors.joining(":")); } - private ServerSocketChannel getChannelFromServiceLayer(int listenPort, BundleContext bundleContext) { - log.log(Level.FINE, "Retrieving channel for port " + listenPort + " from " + bundleContext.getClass().getName()); - Collection<ServiceReference<ServerSocketChannel>> refs; - final String filter = "(port=" + listenPort + ")"; - try { - refs = bundleContext.getServiceReferences(ServerSocketChannel.class, filter); - } catch (InvalidSyntaxException e) { - throw new IllegalStateException("OSGi framework rejected filter " + filter, e); - } - if (refs.isEmpty()) { - return null; - } - if (refs.size() != 1) { - throw new IllegalStateException("Got more than one service reference for " + ServerSocketChannel.class + " port " + listenPort + "."); - } - ServiceReference<ServerSocketChannel> ref = refs.iterator().next(); - return bundleContext.getService(ref); - } - private static ExecutorService newJanitor(ThreadFactory factory) { int threadPoolSize = Runtime.getRuntime().availableProcessors(); log.info("Creating janitor executor with " + threadPoolSize + " threads"); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SecuredRedirectHandler.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SecuredRedirectHandler.java new file mode 100644 index 00000000000..32c0628186a --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SecuredRedirectHandler.java @@ -0,0 +1,52 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.yahoo.jdisc.http.ConnectorConfig; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.HandlerWrapper; +import org.eclipse.jetty.util.URIUtil; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A secure redirect handler inspired by {@link org.eclipse.jetty.server.handler.SecuredRedirectHandler}. + * + * @author bjorncs + */ +class SecuredRedirectHandler extends HandlerWrapper { + + private final Map<Integer, Integer> redirectMap; + + SecuredRedirectHandler(List<ConnectorConfig> connectorConfigs) { + this.redirectMap = createRedirectMap(connectorConfigs); + } + + @Override + public void handle(String target, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException, ServletException { + int localPort = servletRequest.getLocalPort(); + if (!redirectMap.containsKey(localPort)) { + _handler.handle(target, request, servletRequest, servletResponse); + return; + } + servletResponse.setContentLength(0); + servletResponse.sendRedirect( + URIUtil.newURI("https", request.getServerName(), redirectMap.get(localPort), request.getRequestURI(), request.getQueryString())); + request.setHandled(true); + } + + private static Map<Integer, Integer> createRedirectMap(List<ConnectorConfig> connectorConfigs) { + var redirectMap = new HashMap<Integer, Integer>(); + for (ConnectorConfig connectorConfig : connectorConfigs) { + if (connectorConfig.secureRedirect().enabled()) { + redirectMap.put(connectorConfig.listenPort(), connectorConfig.secureRedirect().port()); + } + } + return redirectMap; + } +} diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def index 8027525521c..fa7ed6657d9 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def @@ -106,3 +106,15 @@ proxyProtocol.enabled bool default=false # Allow https in parallel with proxy protocol proxyProtocol.mixedMode bool default=false + +# Redirect all requests to https port +secureRedirect.enabled bool default=false + +# Target port for redirect +secureRedirect.port int default=443 + +# Maximum number of request per connection before server marks connections as non-persistent. Set to '0' to disable. +maxRequestsPerConnection int default=0 + +# Maximum number of seconds a connection can live before it's marked as non-persistent. Set to '0' to disable. +maxConnectionLife double default=0.0 diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def index 0836a080e1f..33f82963243 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def @@ -7,13 +7,16 @@ developerMode bool default=false # The gzip compression level to use, if compression is enabled in a request. responseCompressionLevel int default=6 -# Whether to enable HTTP keep-alive for requests that support this. +# DEPRECATED - Ignored, no longer in use. httpKeepAliveEnabled bool default=true +# TODO Vespa 8 Remove httpKeepAliveEnabled # Maximum number of request per http connection before server will hangup. # Naming taken from apache http server. # 0 means never hangup. +# DEPRECATED - Ignored, no longer in use. Use similar parameter in connector config instead. maxKeepAliveRequests int default=0 +# TODO Vespa 8 Remove maxKeepAliveRequests # Whether the request body of POSTed forms should be removed (form parameters are available as request parameters). removeRawPostBodyForWwwUrlEncodedPost bool default=false diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index 6ace9699b42..f2f3fb0ef11 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -484,8 +484,8 @@ public class HttpServerTest { public void requireThatConnectionIsClosedAfterXRequests() throws Exception { final int MAX_KEEPALIVE_REQUESTS = 100; final TestDriver driver = TestDrivers.newConfiguredInstance(new EchoRequestHandler(), - new ServerConfig.Builder().maxKeepAliveRequests(MAX_KEEPALIVE_REQUESTS), - new ConnectorConfig.Builder()); + new ServerConfig.Builder(), + new ConnectorConfig.Builder().maxRequestsPerConnection(MAX_KEEPALIVE_REQUESTS)); for (int i = 0; i < MAX_KEEPALIVE_REQUESTS - 1; i++) { driver.client().get("/status.html") .expectStatusCode(is(OK)) diff --git a/jrt/src/com/yahoo/jrt/Acceptor.java b/jrt/src/com/yahoo/jrt/Acceptor.java index 9e9dafcbcb5..aed22ac090c 100644 --- a/jrt/src/com/yahoo/jrt/Acceptor.java +++ b/jrt/src/com/yahoo/jrt/Acceptor.java @@ -101,7 +101,7 @@ public class Acceptor { while (serverChannel.isOpen()) { try { TransportThread tt = parent.selectThread(); - tt.addConnection(new Connection(tt, owner, serverChannel.accept())); + tt.addConnection(new Connection(tt, owner, serverChannel.accept(), parent.getTcpNoDelay())); tt.sync(); } catch (ClosedChannelException ignore) { } catch (Exception e) { diff --git a/jrt/src/com/yahoo/jrt/Connection.java b/jrt/src/com/yahoo/jrt/Connection.java index c9c6d78ffba..5a4478cf91e 100644 --- a/jrt/src/com/yahoo/jrt/Connection.java +++ b/jrt/src/com/yahoo/jrt/Connection.java @@ -36,6 +36,7 @@ class Connection extends Target { private final Buffer output = new Buffer(WRITE_SIZE * 2); private int maxInputSize = 64*1024; private int maxOutputSize = 64*1024; + private final boolean tcpNoDelay; private final Map<Integer, ReplyHandler> replyMap = new HashMap<>(); private final Map<TargetWatcher, TargetWatcher> watchers = new IdentityHashMap<>(); private int activeReqs = 0; @@ -89,21 +90,23 @@ class Connection extends Target { } public Connection(TransportThread parent, Supervisor owner, - SocketChannel channel) { + SocketChannel channel, boolean tcpNoDelay) { this.parent = parent; this.owner = owner; this.socket = parent.transport().createServerCryptoSocket(channel); this.spec = null; + this.tcpNoDelay = tcpNoDelay; server = true; owner.sessionInit(this); } - public Connection(TransportThread parent, Supervisor owner, Spec spec, Object context) { + public Connection(TransportThread parent, Supervisor owner, Spec spec, Object context, boolean tcpNoDelay) { super(context); this.parent = parent; this.owner = owner; this.spec = spec; + this.tcpNoDelay = tcpNoDelay; server = false; owner.sessionInit(this); } @@ -184,7 +187,7 @@ class Connection extends Target { } try { socket.channel().configureBlocking(false); - socket.channel().socket().setTcpNoDelay(true); + socket.channel().socket().setTcpNoDelay(tcpNoDelay); selectionKey = socket.channel().register(selector, SelectionKey.OP_READ | SelectionKey.OP_WRITE, this); diff --git a/jrt/src/com/yahoo/jrt/Transport.java b/jrt/src/com/yahoo/jrt/Transport.java index 8abd3942a39..02a6e3e05f7 100644 --- a/jrt/src/com/yahoo/jrt/Transport.java +++ b/jrt/src/com/yahoo/jrt/Transport.java @@ -25,6 +25,7 @@ public class Transport { private final Connector connector; private final Worker worker; private final AtomicInteger runCnt; + private final boolean tcpNoDelay; private final TransportMetrics metrics = TransportMetrics.getInstance(); private final ArrayList<TransportThread> threads = new ArrayList<>(); @@ -40,11 +41,10 @@ public class Transport { * @param cryptoEngine crypto engine to use * @param numThreads number of {@link TransportThread}s. **/ - public Transport(FatalErrorHandler fatalHandler, CryptoEngine cryptoEngine, int numThreads) { - synchronized (this) { - this.fatalHandler = fatalHandler; // NB: this must be set first - } + public Transport(FatalErrorHandler fatalHandler, CryptoEngine cryptoEngine, int numThreads, boolean tcpNoDelay) { + this.fatalHandler = fatalHandler; // NB: this must be set first this.cryptoEngine = cryptoEngine; + this.tcpNoDelay = tcpNoDelay; connector = new Connector(); worker = new Worker(this); runCnt = new AtomicInteger(numThreads); @@ -52,10 +52,10 @@ public class Transport { threads.add(new TransportThread(this)); } } - public Transport(CryptoEngine cryptoEngine, int numThreads) { this(null, cryptoEngine, numThreads); } - public Transport(FatalErrorHandler fatalHandler, int numThreads) { this(fatalHandler, CryptoEngine.createDefault(), numThreads); } - public Transport(int numThreads) { this(null, CryptoEngine.createDefault(), numThreads); } - public Transport() { this(null, CryptoEngine.createDefault(), 1); } + public Transport(CryptoEngine cryptoEngine, int numThreads) { this(null, cryptoEngine, numThreads, true); } + public Transport(int numThreads) { this(null, CryptoEngine.createDefault(), numThreads, true); } + public Transport(int numThreads, boolean tcpNoDelay) { this(null, CryptoEngine.createDefault(), numThreads, tcpNoDelay); } + public Transport() { this(null, CryptoEngine.createDefault(), 1, true); } /** * Select a random transport thread @@ -66,6 +66,8 @@ public class Transport { return threads.get(rnd.nextInt(threads.size())); } + boolean getTcpNoDelay() { return tcpNoDelay; } + /** * Use the underlying CryptoEngine to create a CryptoSocket for * the client side of a connection. @@ -130,7 +132,7 @@ public class Transport { * @param context application context for the new connection */ Connection connect(Supervisor owner, Spec spec, Object context) { - Connection conn = new Connection(selectThread(), owner, spec, context); + Connection conn = new Connection(selectThread(), owner, spec, context, getTcpNoDelay()); connector.connectLater(conn); return conn; } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java index 0fd52e9bdbc..554977d7eb1 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java @@ -67,10 +67,15 @@ public class RPCNetwork implements Network, MethodHandler { new ThreadPoolExecutor(getNumThreads(), getNumThreads(), 0L, TimeUnit.SECONDS, new SynchronousQueue<>(false), ThreadFactoryFactory.getDaemonThreadFactory("mbus.net"), new ThreadPoolExecutor.CallerRunsPolicy()); + private static int getNumThreads() { return Math.max(2, Runtime.getRuntime().availableProcessors()/2); } + private static boolean shouldEnableTcpNodelay(RPCNetworkParams.Optimization optimization) { + return optimization == RPCNetworkParams.Optimization.LATENCY; + } + /** * Create an RPCNetwork. The servicePrefix is combined with session names to create service names. If the service * prefix is 'a/b' and the session name is 'c', the resulting service name that identifies the session on the @@ -82,7 +87,7 @@ public class RPCNetwork implements Network, MethodHandler { public RPCNetwork(RPCNetworkParams params, SlobrokConfigSubscriber slobrokConfig) { this.slobroksConfig = slobrokConfig; identity = params.getIdentity(); - orb = new Supervisor(new Transport(2)); + orb = new Supervisor(new Transport(params.getNumNetworkThreads(), shouldEnableTcpNodelay(params.getOptimization()))); orb.setMaxInputBufferSize(params.getMaxInputBufferSize()); orb.setMaxOutputBufferSize(params.getMaxOutputBufferSize()); targetPool = new RPCTargetPool(params.getConnectionExpireSecs(), params.getNumTargetsPerSpec()); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetworkParams.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetworkParams.java index d6d7603f54a..e77cddd8b06 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetworkParams.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetworkParams.java @@ -20,6 +20,9 @@ public class RPCNetworkParams { private int maxOutputBufferSize = 256 * 1024; private double connectionExpireSecs = 30; private int numTargetsPerSpec = 1; + private int numNetworkThreads = 2; + public enum Optimization {LATENCY, THROUGHPUT} + Optimization optimization = Optimization.LATENCY; /** * Constructs a new instance of this class with reasonable default values. @@ -42,6 +45,8 @@ public class RPCNetworkParams { maxInputBufferSize = params.maxInputBufferSize; maxOutputBufferSize = params.maxOutputBufferSize; numTargetsPerSpec = params.numTargetsPerSpec; + numNetworkThreads = params.numNetworkThreads; + optimization = params.optimization; } /** @@ -152,6 +157,22 @@ public class RPCNetworkParams { return numTargetsPerSpec; } + public RPCNetworkParams setNumNetworkThreads(int numNetworkThreads) { + this.numNetworkThreads = numNetworkThreads; + return this; + } + int getNumNetworkThreads() { + return numNetworkThreads; + } + + public RPCNetworkParams setOptimization(Optimization optimization) { + this.optimization = optimization; + return this; + } + Optimization getOptimization() { + return optimization; + } + /** * Returns the maximum input buffer size allowed for the underlying FNET connection. * diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index c6f61b383bc..4b498c4c014 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -130,6 +130,7 @@ RPCNetwork::RPCNetwork(const RPCNetworkParams ¶ms) : { _transport->SetMaxInputBufferSize(params.getMaxInputBufferSize()); _transport->SetMaxOutputBufferSize(params.getMaxOutputBufferSize()); + _transport->SetTCPNoDelay(params.getTcpNoDelay()); } RPCNetwork::~RPCNetwork() @@ -306,8 +307,8 @@ RPCNetwork::resolveServiceAddress(RoutingNode &recipient, const string &serviceN make_string("Failed to connect to service '%s' from host '%s'.", serviceName.c_str(), getIdentity().getHostname().c_str())); } - ret->setTarget(target); // free by freeServiceAddress() - recipient.setServiceAddress(IServiceAddress::UP(ret.release())); + ret->setTarget(std::move(target)); // free by freeServiceAddress() + recipient.setServiceAddress(std::move(ret)); return Error(); } diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp index bd87e4dbbe2..5bf277a8ee6 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp @@ -15,6 +15,7 @@ RPCNetworkParams::RPCNetworkParams(config::ConfigUri configUri) : _maxInputBufferSize(256*1024), _maxOutputBufferSize(256*1024), _numThreads(4), + _tcpNoDelay(true), _dispatchOnEncode(true), _dispatchOnDecode(false), _connectionExpireSecs(600), diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h index ba530257030..140f81c611c 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h @@ -20,6 +20,7 @@ private: uint32_t _maxInputBufferSize; uint32_t _maxOutputBufferSize; uint32_t _numThreads; + bool _tcpNoDelay; bool _dispatchOnEncode; bool _dispatchOnDecode; double _connectionExpireSecs; @@ -106,6 +107,13 @@ public: uint32_t getNumThreads() const { return _numThreads; } + RPCNetworkParams &setTcpNoDelay(bool tcpNoDelay) { + _tcpNoDelay = tcpNoDelay; + return *this; + } + + bool getTcpNoDelay() const { return _tcpNoDelay; } + /** * Returns the number of seconds before an idle network connection expires. * diff --git a/messagebus/src/vespa/messagebus/network/rpcservice.cpp b/messagebus/src/vespa/messagebus/network/rpcservice.cpp index 6e7c73b38ee..fd1b84f545f 100644 --- a/messagebus/src/vespa/messagebus/network/rpcservice.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcservice.cpp @@ -5,8 +5,7 @@ namespace mbus { -RPCService::RPCService(const Mirror &mirror, - const string &pattern) : +RPCService::RPCService(const Mirror &mirror, const string &pattern) : _mirror(mirror), _pattern(pattern), _addressIdx(random()), @@ -14,7 +13,7 @@ RPCService::RPCService(const Mirror &mirror, _addressList() { } -RPCService::~RPCService() {} +RPCService::~RPCService() = default; RPCServiceAddress::UP RPCService::resolve() @@ -22,9 +21,7 @@ RPCService::resolve() if (_pattern.find("tcp/") == 0) { size_t pos = _pattern.find_last_of('/'); if (pos != string::npos && pos < _pattern.size() - 1) { - RPCServiceAddress::UP ret(new RPCServiceAddress( - _pattern, - _pattern.substr(0, pos))); + auto ret = std::make_unique<RPCServiceAddress>(_pattern, _pattern.substr(0, pos)); if (!ret->isMalformed()) { return ret; } @@ -37,9 +34,7 @@ RPCService::resolve() if (!_addressList.empty()) { _addressIdx = (_addressIdx + 1) % _addressList.size(); const AddressList::value_type &entry = _addressList[_addressIdx]; - return RPCServiceAddress::UP(new RPCServiceAddress( - entry.first, - entry.second)); + return std::make_unique<RPCServiceAddress>(entry.first, entry.second); } } return RPCServiceAddress::UP(); diff --git a/messagebus/src/vespa/messagebus/network/rpcserviceaddress.cpp b/messagebus/src/vespa/messagebus/network/rpcserviceaddress.cpp index eac33195caa..e76832f0620 100644 --- a/messagebus/src/vespa/messagebus/network/rpcserviceaddress.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcserviceaddress.cpp @@ -16,7 +16,7 @@ RPCServiceAddress::RPCServiceAddress(const string &serviceName, } } -RPCServiceAddress::~RPCServiceAddress() {} +RPCServiceAddress::~RPCServiceAddress() = default; bool RPCServiceAddress::isMalformed() diff --git a/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h b/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h index 36dde19bd18..99a9f383e75 100644 --- a/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h +++ b/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h @@ -32,8 +32,7 @@ public: * @param serviceName The full service name of the address. * @param connectionSpec The connection specification. */ - RPCServiceAddress(const string &serviceName, - const string &connectionSpec); + RPCServiceAddress(const string &serviceName, const string &connectionSpec); ~RPCServiceAddress(); /** @@ -69,7 +68,7 @@ public: * * @param target The target to set. */ - void setTarget(RPCTarget::SP target) { _target = target; } + void setTarget(RPCTarget::SP target) { _target = std::move(target); } /** * Returns the RPC target to be used when communicating with the remove service. Make sure that {@link @@ -84,7 +83,7 @@ public: * * @return True if target is set. */ - bool hasTarget() const { return _target.get() != nullptr; } + bool hasTarget() const { return bool(_target); } }; } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp b/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp index b306cf29cf9..fb40ccff62b 100644 --- a/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp @@ -14,17 +14,16 @@ RPCServicePool::RPCServicePool(RPCNetwork &net, uint32_t maxSize) : assert(maxSize > 0); } -RPCServicePool::~RPCServicePool() -{ -} +RPCServicePool::~RPCServicePool() = default; RPCServiceAddress::UP RPCServicePool::resolve(const string &pattern) { - if (_lru.hasKey(pattern)) { - return _lru[pattern]->resolve(); + std::unique_ptr<RPCService> * found = _lru.findAndRef(pattern); + if (found) { + return (*found)->resolve(); } else { - RPCService::UP service(new RPCService(_net.getMirror(), pattern)); + auto service = std::make_unique<RPCService>(_net.getMirror(), pattern); auto result = service->resolve(); _lru[pattern] = std::move(service); return result; diff --git a/messagebus/src/vespa/messagebus/network/rpctarget.cpp b/messagebus/src/vespa/messagebus/network/rpctarget.cpp index 63470b6b707..ea21010e21c 100644 --- a/messagebus/src/vespa/messagebus/network/rpctarget.cpp +++ b/messagebus/src/vespa/messagebus/network/rpctarget.cpp @@ -25,12 +25,14 @@ RPCTarget::~RPCTarget() void RPCTarget::resolveVersion(duration timeout, RPCTarget::IVersionHandler &handler) { - bool hasVersion = false; bool shouldInvoke = false; - { + ResolveState state = _state.load(std::memory_order_acquire); + bool hasVersion = (state == VERSION_RESOLVED); + if ( ! hasVersion ) { vespalib::MonitorGuard guard(_lock); - if (_state == VERSION_RESOLVED || _state == PROCESSING_HANDLERS) { - while (_state == PROCESSING_HANDLERS) { + state = _state.load(std::memory_order_relaxed); + if (state == VERSION_RESOLVED || state == PROCESSING_HANDLERS) { + while (_state.load(std::memory_order::memory_order_relaxed) == PROCESSING_HANDLERS) { guard.wait(); } hasVersion = true; @@ -54,11 +56,11 @@ RPCTarget::resolveVersion(duration timeout, RPCTarget::IVersionHandler &handler) bool RPCTarget::isValid() const { - vespalib::MonitorGuard guard(_lock); if (_target.IsValid()) { return true; } - if (_state == TARGET_INVOKED || _state == PROCESSING_HANDLERS) { + ResolveState state = _state.load(std::memory_order_relaxed); + if (state == TARGET_INVOKED || state == PROCESSING_HANDLERS) { return true; // keep alive until RequestDone() is called } return false; diff --git a/messagebus/src/vespa/messagebus/network/rpctarget.h b/messagebus/src/vespa/messagebus/network/rpctarget.h index b6488f25cb7..d927292f26d 100644 --- a/messagebus/src/vespa/messagebus/network/rpctarget.h +++ b/messagebus/src/vespa/messagebus/network/rpctarget.h @@ -50,13 +50,13 @@ private: }; typedef std::unique_ptr<vespalib::Version> Version_UP; - vespalib::Monitor _lock; - FRT_Supervisor &_orb; - string _name; - FRT_Target &_target; - ResolveState _state; - Version_UP _version; - HandlerList _versionHandlers; + vespalib::Monitor _lock; + FRT_Supervisor &_orb; + string _name; + FRT_Target &_target; + std::atomic<ResolveState> _state; + Version_UP _version; + HandlerList _versionHandlers; public: /** diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp index 7fcc214faa7..cc09e44c460 100644 --- a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp +++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp @@ -63,19 +63,20 @@ RPCTargetPool::size() RPCTarget::SP RPCTargetPool::getTarget(FRT_Supervisor &orb, const RPCServiceAddress &address) { + const string & spec = address.getConnectionSpec(); + uint64_t currentTime = _timer->getMilliTime(); vespalib::LockGuard guard(_lock); - string spec = address.getConnectionSpec(); - TargetMap::iterator it = _targets.find(spec); + auto it = _targets.find(spec); if (it != _targets.end()) { Entry &entry = it->second; if (entry._target->isValid()) { - entry._lastUse = _timer->getMilliTime(); + entry._lastUse = currentTime; return entry._target; } _targets.erase(it); } - RPCTarget::SP ret(new RPCTarget(spec, orb)); - _targets.insert(TargetMap::value_type(spec, Entry(ret, _timer->getMilliTime()))); + auto ret = std::make_shared<RPCTarget>(spec, orb); + _targets.insert(TargetMap::value_type(spec, Entry(ret, currentTime))); return ret; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 01f602d1a57..9c8b0ec2b86 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -52,7 +52,8 @@ public class NodeAgentImpl implements NodeAgent { private static final long BYTES_IN_GB = 1_000_000_000L; // Container is started with uncapped CPU and is kept that way until the first successful health check + this duration - private static final Duration DEFAULT_WARM_UP_DURATION = Duration.ofMinutes(1); + // Subtract 1 second to avoid warmup coming in lockstep with tick time and always end up using an extra tick when there are just a few ms left + private static final Duration DEFAULT_WARM_UP_DURATION = Duration.ofSeconds(90).minus(Duration.ofSeconds(1)); private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName()); @@ -103,7 +104,7 @@ public class NodeAgentImpl implements NodeAgent { FlagSource flagSource, Optional<CredentialsMaintainer> credentialsMaintainer, Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock) { this(contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, credentialsMaintainer, - aclMaintainer, healthChecker, clock, DEFAULT_WARM_UP_DURATION); + aclMaintainer, healthChecker, clock, DEFAULT_WARM_UP_DURATION); } public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository, @@ -211,7 +212,7 @@ public class NodeAgentImpl implements NodeAgent { private Container startContainer(NodeAgentContext context) { ContainerData containerData = createContainerData(context); - ContainerResources wantedResources = context.nodeType() != NodeType.tenant || warmUpDuration.isNegative() ? + ContainerResources wantedResources = context.nodeType() != NodeType.tenant || warmUpDuration(context.zone()).isNegative() ? getContainerResources(context) : getContainerResources(context).withUnlimitedCpus(); dockerOperations.createContainer(context, containerData, wantedResources); dockerOperations.startContainer(context); @@ -357,7 +358,7 @@ public class NodeAgentImpl implements NodeAgent { ContainerResources wantedContainerResources = getContainerResources(context); if (healthChecker.isPresent() && firstSuccessfulHealthCheckInstant - .map(clock.instant().minus(warmUpDuration)::isBefore) + .map(clock.instant().minus(warmUpDuration(context.zone()))::isBefore) .orElse(true)) return existingContainer; @@ -473,7 +474,7 @@ public class NodeAgentImpl implements NodeAgent { if (firstSuccessfulHealthCheckInstant.isEmpty()) firstSuccessfulHealthCheckInstant = Optional.of(clock.instant()); - Duration timeLeft = Duration.between(clock.instant(), firstSuccessfulHealthCheckInstant.get().plus(warmUpDuration)); + Duration timeLeft = Duration.between(clock.instant(), firstSuccessfulHealthCheckInstant.get().plus(warmUpDuration(context.zone()))); if (!container.get().resources.equalsCpu(getContainerResources(context))) throw new ConvergenceException("Refusing to resume until warm up period ends (" + (timeLeft.isNegative() ? "next tick" : "in " + timeLeft) + ")"); @@ -604,4 +605,10 @@ public class NodeAgentImpl implements NodeAgent { protected Optional<CredentialsMaintainer> credentialsMaintainer() { return credentialsMaintainer; } + + private Duration warmUpDuration(ZoneApi zone) { + return zone.getSystemName().isCd() || zone.getEnvironment().isTest() + ? Duration.ofSeconds(-1) + : warmUpDuration; + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java index 83ac3eeeaf4..22cabdc8ed9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java @@ -1,6 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -10,6 +12,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.logging.Logger; import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.ifExists; import static com.yahoo.yolean.Exceptions.uncheck; @@ -23,6 +26,8 @@ import static java.util.stream.Collectors.joining; */ public class DefaultEnvWriter { + private static final Logger logger = Logger.getLogger(DefaultEnvWriter.class.getName()); + private final Map<String, Operation> operations = new LinkedHashMap<>(); public DefaultEnvWriter addOverride(String name, String value) { @@ -50,12 +55,13 @@ public class DefaultEnvWriter { * * @return true if the file was modified */ - public boolean updateFile(Path defaultEnvFile) { + public boolean updateFile(TaskContext context, Path defaultEnvFile) { List<String> currentDefaultEnvLines = ifExists(() -> Files.readAllLines(defaultEnvFile)).orElse(List.of()); List<String> newDefaultEnvLines = generateContent(currentDefaultEnvLines); if (currentDefaultEnvLines.equals(newDefaultEnvLines)) { return false; } else { + context.log(logger, "Updating " + defaultEnvFile.toString()); Path tempFile = Paths.get(defaultEnvFile.toString() + ".tmp"); uncheck(() -> Files.write(tempFile, newDefaultEnvLines)); uncheck(() -> Files.move(tempFile, defaultEnvFile, ATOMIC_MOVE)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 268e0a5ccfd..07e73eb0ee7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -11,6 +11,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileTime; import java.nio.file.attribute.GroupPrincipal; import java.nio.file.attribute.PosixFileAttributeView; import java.nio.file.attribute.PosixFileAttributes; @@ -37,22 +38,33 @@ import static com.yahoo.yolean.Exceptions.uncheck; public class UnixPath { private final Path path; - public UnixPath(Path path) { - this.path = path; - } + public UnixPath(Path path) { this.path = path; } + public UnixPath(String path) { this(Paths.get(path)); } - public UnixPath(String path) { - this(Paths.get(path)); - } + public Path toPath() { return path; } + public UnixPath resolve(String relativeOrAbsolutePath) { return new UnixPath(path.resolve(relativeOrAbsolutePath)); } + + public UnixPath getParent() { + Path parentPath = path.getParent(); + if (parentPath == null) { + throw new IllegalStateException("Path has no parent directory: '" + path + "'"); + } - public Path toPath() { - return path; + return new UnixPath(parentPath); } - public boolean exists() { - return Files.exists(path); + public String getFilename() { + Path filename = path.getFileName(); + if (filename == null) { + // E.g. "/". + throw new IllegalStateException("Path has no filename: '" + path.toString() + "'"); + } + + return filename.toString(); } + public boolean exists() { return Files.exists(path); } + public String readUtf8File() { return new String(readBytes(), StandardCharsets.UTF_8); } @@ -91,6 +103,18 @@ public class UnixPath { return this; } + public UnixPath atomicWriteUt8(String content) { + return atomicWriteBytes(content.getBytes(StandardCharsets.UTF_8)); + } + + /** Write a file to the same dir as this, and then atomically move it to this' path. */ + public UnixPath atomicWriteBytes(byte[] content) { + UnixPath temporaryPath = getParent().resolve(getFilename() + ".10Ia2f4N5"); + temporaryPath.writeBytes(content); + temporaryPath.atomicMove(path); + return this; + } + public String getPermissions() { return getAttributes().permissions(); } @@ -135,6 +159,11 @@ public class UnixPath { return getAttributes().lastModifiedTime(); } + public UnixPath updateLastModifiedTime() { + uncheck(() -> Files.setLastModifiedTime(path, FileTime.from(Instant.now()))); + return this; + } + public FileAttributes getAttributes() { PosixFileAttributes attributes = uncheck(() -> Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java index a4138b215c4..5f7fbdd0d69 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java @@ -69,7 +69,7 @@ public class Yum { * * @return false only if the package was already locked and installed at the given version (no-op) */ - public boolean installFixedVersion(TaskContext context, YumPackageName yumPackage) { + public boolean installFixedVersion(TaskContext context, YumPackageName yumPackage, String... repos) { String targetVersionLockName = yumPackage.toVersionLockName(); boolean alreadyLocked = terminal @@ -118,25 +118,25 @@ public class Yum { // - "Nothing to do" // And in case we need to downgrade and return true from converge() - CommandLine commandLine = terminal - .newCommandLine(context) - .add("yum", "install", "--assumeyes", yumPackage.toName()); + var installCommand = terminal.newCommandLine(context).add("yum", "install"); + for (String repo : repos) installCommand.add("--enablerepo=" + repo); + installCommand.add("--assumeyes", yumPackage.toName()); - String output = commandLine.executeSilently().getUntrimmedOutput(); + String output = installCommand.executeSilently().getUntrimmedOutput(); if (NOTHING_TO_DO_PATTERN.matcher(output).find()) { if (CHECKING_FOR_UPDATE_PATTERN.matcher(output).find()) { // case 3. - terminal.newCommandLine(context) - .add("yum", "downgrade", "--assumeyes", yumPackage.toName()) - .execute(); + var upgradeCommand = terminal.newCommandLine(context).add("yum", "downgrade", "--assumeyes"); + for (String repo : repos) upgradeCommand.add("--enablerepo=" + repo); + upgradeCommand.add(yumPackage.toName()).execute(); modified = true; } else { // case 2. } } else { // case 1. - commandLine.recordSilentExecutionAsSystemModification(); + installCommand.recordSilentExecutionAsSystemModification(); modified = true; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java index 54c8719bceb..3f5c3025850 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java @@ -72,6 +72,15 @@ public class YumPackageName { architecture = packageName.architecture; } + /** + * Set the epoch of the YUM package. + * + * <p>WARNING: Should only be invoked if the YUM package actually has an epoch. Typically + * YUM packages doesn't have one explicitly set, and in case "0" will be used with + * {@link #toVersionLockName()} (otherwise it fails), but it will be absent from an + * install with {@link #toName()} (otherwise it fails). This typically means that + * you should set this only if the epoch is != "0".</p> + */ public Builder setEpoch(String epoch) { this.epoch = Optional.of(epoch); return this; } public Builder setName(String name) { this.name = name; return this; } public Builder setVersion(String version) { this.version = Optional.of(version); return this; } @@ -235,7 +244,7 @@ public class YumPackageName { */ public String toVersionLockName() { return String.format("%s:%s-%s-%s.%s", - epoch.orElseThrow(() -> new IllegalStateException("Epoch is missing for YUM package " + name)), + epoch.orElse("0"), name, version.orElseThrow(() -> new IllegalStateException("Version is missing for YUM package " + name)), release.orElseThrow(() -> new IllegalStateException("Release is missing for YUM package " + name)), diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java index a81ad8ff2eb..a2457266560 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -9,11 +10,16 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.logging.Logger; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; /** * @author bjorncs @@ -26,6 +32,8 @@ public class DefaultEnvWriterTest { private static final Path EXAMPLE_FILE = Paths.get("src/test/resources/default-env-example.txt"); private static final Path EXPECTED_RESULT_FILE = Paths.get("src/test/resources/default-env-rewritten.txt"); + private final TaskContext context = mock(TaskContext.class); + @Test public void default_env_is_correctly_rewritten() throws IOException { Path tempFile = temporaryFolder.newFile().toPath(); @@ -36,14 +44,16 @@ public class DefaultEnvWriterTest { writer.addFallback("VESPA_CONFIGSERVER", "new-fallback-configserver"); writer.addOverride("VESPA_TLS_CONFIG_FILE", "/override/path/to/config.file"); - boolean modified = writer.updateFile(tempFile); + boolean modified = writer.updateFile(context, tempFile); assertTrue(modified); assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile)); + verify(context, times(1)).log(any(Logger.class), any(String.class)); - modified = writer.updateFile(tempFile); + modified = writer.updateFile(context, tempFile); assertFalse(modified); assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile)); + verify(context, times(1)).log(any(Logger.class), any(String.class)); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java index 3b839f7f446..3159689c22e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.ComparisonFailure; import org.junit.Test; import java.nio.file.FileSystem; @@ -13,6 +14,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author hakonhall @@ -124,4 +126,44 @@ public class UnixPathTest { assertFalse(dir1 + " deleted recursively", Files.exists(dir1)); } + @Test + public void atomicWrite() { + var path = new UnixPath(fs.getPath("/dir/foo")); + path.createParents(); + path.writeUtf8File("bar"); + path.atomicWriteUt8("bar v2"); + assertEquals("bar v2", path.readUtf8File()); + } + + @Test + public void testParentAndFilename() { + var absolutePath = new UnixPath("/foo/bar"); + assertEquals("/foo", absolutePath.getParent().toString()); + assertEquals("bar", absolutePath.getFilename()); + + var pathWithoutSlash = new UnixPath("foo"); + assertRuntimeException(IllegalStateException.class, "Path has no parent directory: 'foo'", () -> pathWithoutSlash.getParent()); + assertEquals("foo", pathWithoutSlash.getFilename()); + + var pathWithSlash = new UnixPath("/foo"); + assertEquals("/", pathWithSlash.getParent().toString()); + assertEquals("foo", pathWithSlash.getFilename()); + + assertRuntimeException(IllegalStateException.class, "Path has no parent directory: '/'", () -> new UnixPath("/").getParent()); + assertRuntimeException(IllegalStateException.class, "Path has no filename: '/'", () -> new UnixPath("/").getFilename()); + } + + private <T extends RuntimeException> void assertRuntimeException(Class<T> baseClass, String message, Runnable runnable) { + try { + runnable.run(); + fail("No exception was thrown"); + } catch (RuntimeException e) { + if (!baseClass.isInstance(e)) { + throw new ComparisonFailure("Exception class mismatch", baseClass.getName(), e.getClass().getName()); + } + + assertEquals(message, e.getMessage()); + } + } + } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java index 01664f5c22b..64e2997d486 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java @@ -56,7 +56,7 @@ public class YumPackageNameTest { "1.el7", null, "docker-engine-selinux-1.12.6-1.el7", - null); + "0:docker-engine-selinux-1.12.6-1.el7.*"); // name-ver-rel.arch verifyPackageName("docker-engine-selinux-1.12.6-1.el7.x86_64", @@ -66,7 +66,7 @@ public class YumPackageNameTest { "1.el7", "x86_64", "docker-engine-selinux-1.12.6-1.el7.x86_64", - null); + "0:docker-engine-selinux-1.12.6-1.el7.*"); // name-epoch:ver-rel.arch verifyPackageName( @@ -112,7 +112,7 @@ public class YumPackageNameTest { yumPackageName.toVersionLockName(); fail(); } catch (IllegalStateException e) { - assertThat(e.getMessage(), containsStringIgnoringCase("epoch is missing")); + assertThat(e.getMessage(), containsStringIgnoringCase("Version is missing ")); } } else { assertEquals(toVersionName, yumPackageName.toVersionLockName()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 3cf2442f6f7..77fa7dfc7e4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -19,6 +19,7 @@ import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; @@ -98,6 +99,7 @@ public class NodeRepository extends AbstractComponent { private final FirmwareChecks firmwareChecks; private final DockerImages dockerImages; private final JobControl jobControl; + private final Applications applications; /** * Creates a node repository from a zookeeper provider. @@ -124,6 +126,7 @@ public class NodeRepository extends AbstractComponent { this.firmwareChecks = new FirmwareChecks(db, clock); this.dockerImages = new DockerImages(db, dockerImage); this.jobControl = new JobControl(db); + this.applications = new Applications(); // read and write all nodes to make sure they are stored in the latest version of the serialized format for (State state : State.values()) @@ -154,6 +157,9 @@ public class NodeRepository extends AbstractComponent { /** Returns the status of maintenance jobs managed by this. */ public JobControl jobControl() { return jobControl; } + /** Returns this node repo's view of the applications deployed to it */ + public Applications applications() { return applications; } + // ---------------- Query API ---------------------------------------------------------------- /** @@ -192,6 +198,11 @@ public class NodeRepository extends AbstractComponent { return NodeList.copyOf(getNodes()); } + /** Returns a filterable list of all nodes of an application */ + public NodeList list(ApplicationId application) { + return NodeList.copyOf(getNodes(application)); + } + /** Returns a locked list of all nodes in this repository */ public LockedNodeList list(Mutex lock) { return new LockedNodeList(getNodes(), lock); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java new file mode 100644 index 00000000000..e56e426b499 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -0,0 +1,59 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.applications; + +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.transaction.Mutex; + +import java.util.Map; +import java.util.HashMap; +import java.util.Optional; + +/** + * The node repository's view of an application deployment. + * + * This is immutable, and must be locked with the application lock on read-modify-write. + * + * @author bratseth + */ +public class Application { + + private final Map<ClusterSpec.Id, Cluster> clusters; + + public Application() { + this(Map.of()); + } + + private Application(Map<ClusterSpec.Id, Cluster> clusters) { + this.clusters = Map.copyOf(clusters); + } + + /** Returns the cluster with the given id or null if none */ + public Cluster cluster(ClusterSpec.Id id) { return clusters.get(id); } + + public Application with(ClusterSpec.Id id, Cluster cluster) { + Map<ClusterSpec.Id, Cluster> clusters = new HashMap<>(this.clusters); + clusters.put(id, cluster); + return new Application(clusters); + } + + /** + * Returns an application with the given cluster having the min and max resource limits of the given cluster. + * If the cluster has a target which is not inside the new limits, the target is removed. + */ + public Application withClusterLimits(ClusterSpec.Id id, ClusterResources min, ClusterResources max) { + Cluster cluster = clusters.get(id); + return with(id, new Cluster(min, max, cluster == null ? Optional.empty() : cluster.targetResources())); + } + + /** + * Returns an application with the given target for the given cluster, + * if it exists and the target is within the bounds + */ + public Application withClusterTarget(ClusterSpec.Id id, ClusterResources target) { + Cluster cluster = clusters.get(id); + if (cluster == null) return this; + return with(id, cluster.withTarget(target)); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java new file mode 100644 index 00000000000..879fcc5f6cb --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java @@ -0,0 +1,29 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.applications; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.transaction.Mutex; + +import java.util.concurrent.ConcurrentHashMap; + +/** + * An (in-memory, for now) repository of the node repo's view of applications. + * + * This is multithread safe. + * + * @author bratseth + */ +public class Applications { + + private final ConcurrentHashMap<ApplicationId, Application> applications = new ConcurrentHashMap<>(); + + /** Returns the application with the given id, or null if it does not exist and should not be created */ + public Application get(ApplicationId applicationId, boolean create) { + return applications.computeIfAbsent(applicationId, id -> create ? new Application() : null); + } + + public void set(ApplicationId id, Application application, Mutex applicationLock) { + applications.put(id, application); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java new file mode 100644 index 00000000000..6ff7f41be8f --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -0,0 +1,66 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.applications; + +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.NodeResources; + +import java.util.Objects; +import java.util.Optional; + +/** + * The node repo's view of a cluster in an application deployment. + * + * This is immutable, and must be locked with the application lock on read-modify-write. + * + * @author bratseth + */ +public class Cluster { + + private final ClusterResources min, max; + private final Optional<ClusterResources> target; + + Cluster(ClusterResources minResources, ClusterResources maxResources, Optional<ClusterResources> targetResources) { + this.min = Objects.requireNonNull(minResources); + this.max = Objects.requireNonNull(maxResources); + Objects.requireNonNull(targetResources); + + if (targetResources.isPresent() && ! targetResources.get().isWithin(minResources, maxResources)) + this.target = Optional.empty(); + else + this.target = targetResources; + } + + /** Returns the configured minimal resources in this cluster */ + public ClusterResources minResources() { return min; } + + /** Returns the configured maximal resources in this cluster */ + public ClusterResources maxResources() { return max; } + + /** + * Returns the computed resources (between min and max, inclusive) this cluster should + * have allocated at the moment, or empty if the system currently have no opinion on this. + */ + public Optional<ClusterResources> targetResources() { return target; } + + public Cluster withTarget(ClusterResources target) { + return new Cluster(min, max, Optional.of(target)); + } + + public Cluster withoutTarget() { + return new Cluster(min, max, Optional.empty()); + } + + public NodeResources capAtLimits(NodeResources resources) { + resources = resources.withVcpu(between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), resources.vcpu())); + resources = resources.withMemoryGb(between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), resources.memoryGb())); + resources = resources.withDiskGb(between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), resources.diskGb())); + return resources; + } + + private double between(double min, double max, double value) { + value = Math.max(min, value); + value = Math.min(max, value); + return value; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index 40af5f43312..5ca09ddf51c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; @@ -14,6 +15,11 @@ import java.util.List; */ public class AllocatableClusterResources { + // We only depend on the ratios between these values + private static final double cpuUnitCost = 12.0; + private static final double memoryUnitCost = 1.2; + private static final double diskUnitCost = 0.045; + /** The node count in the cluster */ private final int nodes; @@ -25,28 +31,40 @@ public class AllocatableClusterResources { private final ClusterSpec.Type clusterType; + private final double fulfilment; + public AllocatableClusterResources(List<Node> nodes, HostResourcesCalculator calculator) { this.advertisedResources = nodes.get(0).flavor().resources(); this.realResources = calculator.realResourcesOf(nodes.get(0)); this.nodes = nodes.size(); this.groups = (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); this.clusterType = nodes.get(0).allocation().get().membership().cluster().type(); + this.fulfilment = 1; } - public AllocatableClusterResources(ClusterResources realResources, NodeResources advertisedResources) { + public AllocatableClusterResources(ClusterResources realResources, + NodeResources advertisedResources, + NodeResources idealResources, + ClusterSpec.Type clusterType) { this.realResources = realResources.nodeResources(); this.advertisedResources = advertisedResources; this.nodes = realResources.nodes(); this.groups = realResources.groups(); - this.clusterType = realResources.clusterType(); + this.clusterType = clusterType; + this.fulfilment = fulfilment(realResources.nodeResources(), idealResources); } - public AllocatableClusterResources(ClusterResources realResources, Flavor flavor, HostResourcesCalculator calculator) { + public AllocatableClusterResources(ClusterResources realResources, + Flavor flavor, + NodeResources idealResources, + ClusterSpec.Type clusterType, + HostResourcesCalculator calculator) { this.realResources = realResources.nodeResources(); this.advertisedResources = calculator.advertisedResourcesOf(flavor); this.nodes = realResources.nodes(); this.groups = realResources.groups(); - this.clusterType = realResources.clusterType(); + this.clusterType = clusterType; + this.fulfilment = fulfilment(realResources.nodeResources(), idealResources); } /** @@ -61,15 +79,46 @@ public class AllocatableClusterResources { */ public NodeResources advertisedResources() { return advertisedResources; } - public double cost() { return nodes * Autoscaler.costOf(advertisedResources); } + public ClusterResources toAdvertisedClusterResources() { + return new ClusterResources(nodes, groups, advertisedResources); + } public int nodes() { return nodes; } public int groups() { return groups; } public ClusterSpec.Type clusterType() { return clusterType; } + public double cost() { return nodes * costOf(advertisedResources); } + + /** + * Returns the fraction measuring how well the real resources fulfils the ideal: 1 means completely fulfiled, + * 0 means we have zero real resources. + * The real may be short of the ideal due to resource limits imposed by the system or application. + */ + public double fulfilment() { return fulfilment; } + + private static double costOf(NodeResources resources) { + return resources.vcpu() * cpuUnitCost + + resources.memoryGb() * memoryUnitCost + + resources.diskGb() * diskUnitCost; + } + + private static double fulfilment(NodeResources realResources, NodeResources idealResources) { + double vcpuFulfilment = Math.min(1, realResources.vcpu() / idealResources.vcpu()); + double memoryGbFulfilment = Math.min(1, realResources.memoryGb() / idealResources.memoryGb()); + double diskGbFulfilment = Math.min(1, realResources.diskGb() / idealResources.diskGb()); + return (vcpuFulfilment + memoryGbFulfilment + diskGbFulfilment) / 3; + } + + public boolean preferableTo(AllocatableClusterResources other) { + if (this.fulfilment > other.fulfilment) return true; // we always want to fulfil as much as possible + return this.cost() < other.cost(); // otherwise, prefer lower cost + } + @Override public String toString() { - return "$" + cost() + ": " + realResources(); + return nodes + " nodes with " + realResources() + + " at cost $" + cost() + + (fulfilment < 1.0 ? " (fulfilment " + fulfilment + ")" : ""); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 0e70178f71e..447d3494fbc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -2,12 +2,14 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; @@ -35,15 +37,10 @@ public class Autoscaler { private static final int minimumMeasurements = 500; // TODO: Per node instead? Also say something about interval? - /** What cost difference factor warrants reallocation? */ - private static final double costDifferenceRatioWorthReallocation = 0.1; - /** What difference factor from ideal (for any resource) warrants a change? */ - private static final double idealDivergenceWorthReallocation = 0.1; - - // We only depend on the ratios between these values - private static final double cpuUnitCost = 12.0; - private static final double memoryUnitCost = 1.2; - private static final double diskUnitCost = 0.045; + /** What cost difference factor is worth a reallocation? */ + private static final double costDifferenceWorthReallocation = 0.1; + /** What difference factor for a resource is worth a reallocation? */ + private static final double resourceDifferenceWorthReallocation = 0.1; private final HostResourcesCalculator resourcesCalculator; private final NodeMetricsDb metricsDb; @@ -65,12 +62,8 @@ public class Autoscaler { * @param clusterNodes the list of all the active nodes in a cluster * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time */ - public Optional<AllocatableClusterResources> autoscale(List<Node> clusterNodes) { - if (clusterNodes.stream().anyMatch(node -> node.status().wantToRetire() || - node.allocation().get().membership().retired() || - node.allocation().get().isRemovable())) { - return Optional.empty(); // Don't autoscale clusters that are in flux - } + public Optional<AllocatableClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes) { + if (unstable(clusterNodes)) return Optional.empty(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, resourcesCalculator); @@ -82,37 +75,38 @@ public class Autoscaler { Optional<AllocatableClusterResources> bestAllocation = findBestAllocation(cpuLoad.get(), memoryLoad.get(), diskLoad.get(), - currentAllocation); + currentAllocation, + cluster); if (bestAllocation.isEmpty()) return Optional.empty(); - - if (closeToIdeal(Resource.cpu, cpuLoad.get()) && - closeToIdeal(Resource.memory, memoryLoad.get()) && - closeToIdeal(Resource.disk, diskLoad.get()) && - similarCost(bestAllocation.get().cost(), currentAllocation.cost())) { - return Optional.empty(); // Avoid small, unnecessary changes - } + if (similar(bestAllocation.get(), currentAllocation)) return Optional.empty(); return bestAllocation; } private Optional<AllocatableClusterResources> findBestAllocation(double cpuLoad, double memoryLoad, double diskLoad, - AllocatableClusterResources currentAllocation) { + AllocatableClusterResources currentAllocation, + Cluster cluster) { Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); - for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation); i.hasNext(); ) { - ClusterResources allocation = i.next(); - Optional<AllocatableClusterResources> allocatableResources = toAllocatableResources(allocation); + for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation, cluster); i.hasNext(); ) { + Optional<AllocatableClusterResources> allocatableResources = toAllocatableResources(i.next(), + currentAllocation.clusterType(), + cluster); if (allocatableResources.isEmpty()) continue; - if (bestAllocation.isEmpty() || allocatableResources.get().cost() < bestAllocation.get().cost()) + if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) bestAllocation = allocatableResources; } return bestAllocation; } - private boolean similarCost(double cost1, double cost2) { - return similar(cost1, cost2, costDifferenceRatioWorthReallocation); - } - - private boolean closeToIdeal(Resource resource, double value) { - return similar(resource.idealAverageLoad(), value, idealDivergenceWorthReallocation); + /** Returns true if both total real resources and total cost are similar */ + private boolean similar(AllocatableClusterResources a, AllocatableClusterResources b) { + return similar(a.cost(), b.cost(), costDifferenceWorthReallocation) && + similar(a.realResources().vcpu() * a.nodes(), + b.realResources().vcpu() * b.nodes(), resourceDifferenceWorthReallocation) && + similar(a.realResources().memoryGb() * a.nodes(), + b.realResources().memoryGb() * b.nodes(), resourceDifferenceWorthReallocation) && + similar(a.realResources().diskGb() * a.nodes(), + b.realResources().diskGb() * b.nodes(), + resourceDifferenceWorthReallocation); } private boolean similar(double r1, double r2, double threshold) { @@ -123,15 +117,22 @@ public class Autoscaler { * Returns the smallest allocatable node resources larger than the given node resources, * or empty if none available. */ - private Optional<AllocatableClusterResources> toAllocatableResources(ClusterResources resources) { - NodeResources nodeResources = nodeResourceLimits.enlargeToLegal(resources.nodeResources(), - resources.clusterType()); + private Optional<AllocatableClusterResources> toAllocatableResources(ClusterResources resources, + ClusterSpec.Type clusterType, + Cluster cluster) { + NodeResources nodeResources = resources.nodeResources(); + if ( ! cluster.minResources().equals(cluster.maxResources())) // enforce application limits unless suggest mode + nodeResources = cluster.capAtLimits(nodeResources); + nodeResources = nodeResourceLimits.enlargeToLegal(nodeResources, clusterType); // enforce system limits + if (allowsHostSharing(nodeRepository.zone().cloud())) { // return the requested resources, or empty if they cannot fit on existing hosts for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { if (flavor.resources().satisfies(nodeResources)) return Optional.of(new AllocatableClusterResources(resources.with(nodeResources), - nodeResources)); + nodeResources, + resources.nodeResources(), + clusterType)); } return Optional.empty(); } @@ -145,6 +146,8 @@ public class Autoscaler { flavor = flavor.with(FlavorOverrides.ofDisk(nodeResources.diskGb())); var candidate = new AllocatableClusterResources(resources.with(flavor.resources()), flavor, + resources.nodeResources(), + clusterType, resourcesCalculator); if (best.isEmpty() || candidate.cost() <= best.get().cost()) @@ -181,10 +184,10 @@ public class Autoscaler { return true; } - static double costOf(NodeResources resources) { - return resources.vcpu() * cpuUnitCost + - resources.memoryGb() * memoryUnitCost + - resources.diskGb() * diskUnitCost; + public static boolean unstable(List<Node> nodes) { + return nodes.stream().anyMatch(node -> node.status().wantToRetire() || + node.allocation().get().membership().retired() || + node.allocation().get().isRemovable()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java deleted file mode 100644 index ebceba8c97f..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.vespa.hosted.provision.Node; - -import java.util.List; -import java.util.Objects; - -/** A description of the resources of a cluster */ -public class ClusterResources { - - /** The node count in the cluster */ - private final int nodes; - - /** The number of node groups in the cluster */ - private final int groups; - - /** The resources of each node in the cluster */ - private final NodeResources nodeResources; - - /** The kind of cluster these resources are for */ - private final ClusterSpec.Type clusterType; - - public ClusterResources(int nodes, int groups, NodeResources nodeResources, ClusterSpec.Type clusterType) { - this.nodes = nodes; - this.groups = groups; - this.nodeResources = nodeResources; - this.clusterType = clusterType; - } - - /** Returns the total number of allocated nodes (over all groups) */ - public int nodes() { return nodes; } - public int groups() { return groups; } - public NodeResources nodeResources() { return nodeResources; } - public ClusterSpec.Type clusterType() { return clusterType; } - - public ClusterResources with(NodeResources resources) { - return new ClusterResources(nodes, groups, resources, clusterType); - } - - @Override - public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof ClusterResources)) return false; - - ClusterResources other = (ClusterResources)o; - if (other.nodes != this.nodes) return false; - if (other.groups != this.groups) return false; - if (other.nodeResources != this.nodeResources) return false; - if (other.clusterType != this.clusterType) return false; - return true; - } - - @Override - public int hashCode() { - return Objects.hash(nodes, groups, nodeResources, clusterType); - } - - @Override - public String toString() { - return clusterType + " cluster resources: " + nodes + " * " + nodeResources + (groups > 1 ? " in " + groups + " groups" : ""); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java index 7137b69b9ac..232fee1df6a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java @@ -7,6 +7,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; import com.yahoo.vespa.orchestrator.Orchestrator; @@ -51,13 +52,15 @@ public class NodeMetricsFetcher extends AbstractComponent implements NodeMetrics @Override public Collection<MetricValue> fetchMetrics(ApplicationId application) { - Optional<Node> metricsV2Container = nodeRepository.list() - .owner(application) - .state(Node.State.active) - .container() - .filter(node -> expectedUp(node)) - .stream() - .findFirst(); + NodeList applicationNodes = nodeRepository.list(application).state(Node.State.active); + + // Do not try to draw conclusions from utilization while unstable + if (Autoscaler.unstable(applicationNodes.asList())) return Collections.emptyList(); + + Optional<Node> metricsV2Container = applicationNodes.container() + .filter(node -> expectedUp(node)) + .stream() + .findFirst(); if (metricsV2Container.isEmpty()) return Collections.emptyList(); String url = "http://" + metricsV2Container.get().hostname() + ":" + 4080 + apiPath + "?consumer=default"; String response = httpClient.get(url); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java index e84544e7e7b..3d5ce8881e0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java @@ -12,7 +12,7 @@ public enum Resource { /** Cpu utilization ratio */ cpu { - String metricName() { return "cpu.util"; } + public String metricName() { return "cpu.util"; } double idealAverageLoad() { return 0.2; } double valueFrom(NodeResources resources) { return resources.vcpu(); } double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio @@ -20,7 +20,7 @@ public enum Resource { /** Memory utilization ratio */ memory { - String metricName() { return "mem_total.util"; } + public String metricName() { return "mem_total.util"; } double idealAverageLoad() { return 0.7; } double valueFrom(NodeResources resources) { return resources.memoryGb(); } double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio @@ -28,13 +28,13 @@ public enum Resource { /** Disk utilization ratio */ disk { - String metricName() { return "disk.util"; } + public String metricName() { return "disk.util"; } double idealAverageLoad() { return 0.6; } double valueFrom(NodeResources resources) { return resources.diskGb(); } double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio }; - abstract String metricName(); + public abstract String metricName(); /** The load we should have of this resource on average, when one node in the cluster is down */ abstract double idealAverageLoad(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java index ee1af65753a..bc14ca1779c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java @@ -1,7 +1,9 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.applications.Cluster; /** * Provides iteration over possible cluster resource allocations given a target total load @@ -9,16 +11,19 @@ import com.yahoo.config.provision.NodeResources; */ public class ResourceIterator { - // Configured min and max nodes TODO: These should come from the application package - private static final int minimumNodesPerCluster = 3; // Since this is with redundancy it cannot be lower than 2 - private static final int maximumNodesPerCluster = 150; + // Configured min and max nodes for suggestions for apps which have not activated autoscaling + private static final int minimumNodes = 3; // Since this is with redundancy it cannot be lower than 2 + private static final int maximumNodes = 150; // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component // proportional to document count. We must account for this when comparing configurations with more or fewer nodes. // TODO: Measure this, and only take it into account with queries private static final double fixedCpuCostFraction = 0.1; - // Describes the observed state + // Prescribed state + private final Cluster cluster; + + // Observed state private final AllocatableClusterResources allocation; private final double cpuLoad; private final double memoryLoad; @@ -32,7 +37,9 @@ public class ResourceIterator { // Iterator state private int currentNodes; - public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad, AllocatableClusterResources currentAllocation) { + public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad, + AllocatableClusterResources currentAllocation, + Cluster cluster) { this.cpuLoad = cpuLoad; this.memoryLoad = memoryLoad; this.diskLoad = diskLoad; @@ -41,6 +48,8 @@ public class ResourceIterator { groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups()); allocation = currentAllocation; + this.cluster = cluster; + // What number of nodes is it effective to add or remove at the time from this cluster? // This is the group size, since we (for now) assume the group size is decided by someone wiser than us // and we decide the number of groups. @@ -48,31 +57,52 @@ public class ResourceIterator { singleGroupMode = currentAllocation.groups() == 1; nodeIncrement = singleGroupMode ? 1 : groupSize; + // Step down to the right starting point currentNodes = currentAllocation.nodes(); - while (currentNodes - nodeIncrement >= minimumNodesPerCluster - && (singleGroupMode || currentNodes - nodeIncrement > groupSize)) // group level redundancy + while (currentNodes - nodeIncrement >= minNodes() + && ( singleGroupMode || currentNodes - nodeIncrement > groupSize)) // group level redundancy currentNodes -= nodeIncrement; } + /** If autoscaling is not enabled (meaning max and min resources are the same), we want to suggest */ + private boolean suggestMode() { + return cluster.minResources().equals(cluster.maxResources()); + } + public ClusterResources next() { - int nodesWithRedundancy = currentNodes - (singleGroupMode ? 1 : groupSize); - ClusterResources next = new ClusterResources(currentNodes, - singleGroupMode ? 1 : currentNodes / groupSize, - resourcesFor(nodesWithRedundancy), - allocation.clusterType()); + ClusterResources next = resourcesWith(currentNodes); currentNodes += nodeIncrement; return next; } public boolean hasNext() { - return currentNodes <= maximumNodesPerCluster; + return currentNodes <= maxNodes(); + } + + private int minNodes() { + if (suggestMode()) return minimumNodes; + if (singleGroupMode) return cluster.minResources().nodes(); + return Math.max(cluster.minResources().nodes(), cluster.minResources().groups() * groupSize ); + } + + private int maxNodes() { + if (suggestMode()) return maximumNodes; + if (singleGroupMode) return cluster.maxResources().nodes(); + return Math.min(cluster.maxResources().nodes(), cluster.maxResources().groups() * groupSize ); + } + + private ClusterResources resourcesWith(int nodes) { + int nodesWithRedundancy = nodes - (singleGroupMode ? 1 : groupSize); + return new ClusterResources(nodes, + singleGroupMode ? 1 : nodes / groupSize, + nodeResourcesWith(nodesWithRedundancy)); } /** * For the observed load this instance is initialized with, returns the resources needed per node to be at * ideal load given a target node count */ - private NodeResources resourcesFor(int nodeCount) { + private NodeResources nodeResourcesWith(int nodeCount) { // Cpu: Scales with cluster size (TODO: Only reads, writes scales with group size) // Memory and disk: Scales with group size @@ -103,6 +133,7 @@ public class ResourceIterator { disk = nodeUsage(Resource.disk, diskLoad) / Resource.disk.idealAverageLoad(); } } + return allocation.realResources().withVcpu(cpu).withMemoryGb(memory).withDiskGb(disk); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index d4a237f8e23..abfe65408b6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -8,9 +8,10 @@ import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; -import com.yahoo.vespa.hosted.provision.autoscale.ClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; @@ -53,29 +54,60 @@ public class AutoscalingMaintainer extends Maintainer { private void autoscale(ApplicationId application, List<Node> applicationNodes) { try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, nodeRepository())) { if ( ! deployment.isValid()) return; // Another config server will consider this application - nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId, clusterNodes)); + nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId, clusterNodes, deployment)); } } - private void autoscale(ApplicationId application, ClusterSpec.Id clusterId, List<Node> clusterNodes) { - Optional<AllocatableClusterResources> target = autoscaler.autoscale(clusterNodes); - if (target.isEmpty()) return; + private void autoscale(ApplicationId applicationId, + ClusterSpec.Id clusterId, + List<Node> clusterNodes, + MaintenanceDeployment deployment) { + Application application = nodeRepository().applications().get(applicationId, true); + Cluster cluster = application.cluster(clusterId); + if (cluster == null) return; // no information on limits for this cluster + Optional<AllocatableClusterResources> target = autoscaler.autoscale(cluster, clusterNodes); + if (target.isEmpty()) return; // current resources are fine + if (cluster.minResources().equals(cluster.maxResources())) { // autoscaling is deactivated + logAutoscaling("Scaling suggestion for ", target.get(), applicationId, clusterId, clusterNodes); + } + else { + logAutoscaling("Autoscaling ", target.get(), applicationId, clusterId, clusterNodes); + autoscaleTo(target.get(), applicationId, clusterId, application, deployment); + } + } + + private void autoscaleTo(AllocatableClusterResources target, + ApplicationId applicationId, + ClusterSpec.Id clusterId, + Application application, + MaintenanceDeployment deployment) { + nodeRepository().applications().set(applicationId, + application.withClusterTarget(clusterId, target.toAdvertisedClusterResources()), + deployment.applicationLock().get()); + deployment.activate(); + } + + private void logAutoscaling(String prefix, + AllocatableClusterResources target, + ApplicationId application, + ClusterSpec.Id clusterId, + List<Node> clusterNodes) { Instant lastLogTime = lastLogged.get(new Pair<>(application, clusterId)); if (lastLogTime != null && lastLogTime.isAfter(nodeRepository().clock().instant().minus(Duration.ofHours(1)))) return; - int currentGroups = (int) clusterNodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); + int currentGroups = (int)clusterNodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); - log.info("Autoscale: " + application + " " + clusterType + " " + clusterId + + log.info(prefix + application + " " + clusterType + " " + clusterId + ":" + "\nfrom " + toString(clusterNodes.size(), currentGroups, clusterNodes.get(0).flavor().resources()) + - "\nto " + toString(target.get().nodes(), target.get().groups(), target.get().advertisedResources())); + "\nto " + toString(target.nodes(), target.groups(), target.advertisedResources())); lastLogged.put(new Pair<>(application, clusterId), nodeRepository().clock().instant()); } private String toString(int nodes, int groups, NodeResources resources) { return String.format(nodes + (groups > 1 ? " (in " + groups + " groups)" : "") + - " * [vcpu: %1$.1f, memory: %2$.1f Gb, disk %3$.1f Gb]" + - " (total: [vcpu: %4$.1f, memory: %5$.1f Gb, disk: %6$.1f Gb])," + + " * [vcpu: %0$.1f, memory: %1$.1f Gb, disk %2$.1f Gb]" + + " (total: [vcpu: %3$.1f, memory: %4$.1f Gb, disk: %5$.1f Gb])", resources.vcpu(), resources.memoryGb(), resources.diskGb(), nodes * resources.vcpu(), nodes * resources.memoryGb(), nodes * resources.diskGb()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java index 856de2609be..d9e06f87db7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java @@ -36,10 +36,8 @@ class MaintenanceDeployment implements Closeable { public MaintenanceDeployment(ApplicationId application, Deployer deployer, NodeRepository nodeRepository) { this.application = application; Optional<Mutex> lock = tryLock(application, nodeRepository); - try { deployment = tryDeployment(lock, application, deployer, nodeRepository); - this.lock = lock; lock = Optional.empty(); } finally { @@ -52,6 +50,16 @@ class MaintenanceDeployment implements Closeable { return deployment.isPresent(); } + /** + * Returns the application lock held by this, or empty if it is not held. + * + * @throws IllegalStateException id this is called when closed + */ + public Optional<Mutex> applicationLock() { + if (closed) throw new IllegalStateException(this + " is closed"); + return lock; + } + public boolean prepare() { return doStep(() -> deployment.get().prepare()); } @@ -61,7 +69,7 @@ class MaintenanceDeployment implements Closeable { } private boolean doStep(Runnable action) { - if (closed) throw new IllegalStateException("Deployment of '" + application + "' is closed"); + if (closed) throw new IllegalStateException(this + "' is closed"); if ( ! isValid()) return false; try { action.run(); @@ -101,4 +109,9 @@ class MaintenanceDeployment implements Closeable { closed = true; } + @Override + public String toString() { + return "deployment of " + application; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index e107abf8fbb..098d706bf05 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -29,6 +29,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import static com.yahoo.config.provision.NodeResources.DiskSpeed.any; +import static com.yahoo.vespa.hosted.provision.Node.State.active; /** * @author oyving @@ -249,16 +250,16 @@ public class MetricsReporter extends Maintainer { } private static NodeResources getCapacityTotal(NodeList nodes) { - return nodes.nodeType(NodeType.host).asList().stream() + return nodes.nodeType(NodeType.host).state(active).asList().stream() .map(host -> host.flavor().resources()) - .map(resources -> resources.justNumbers()) + .map(NodeResources::justNumbers) .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add); } private static NodeResources getFreeCapacityTotal(NodeList nodes) { - return nodes.nodeType(NodeType.host).asList().stream() + return nodes.nodeType(NodeType.host).state(active).asList().stream() .map(n -> freeCapacityOf(nodes, n)) - .map(resources -> resources.justNumbers()) + .map(NodeResources::justNumbers) .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 94f8fb29b9a..648bf52f455 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -31,46 +31,42 @@ public class CapacityPolicies { this.isUsingAdvertisedResources = zone.cloud().value().equals("aws"); } - public int decideSize(Capacity capacity, ClusterSpec cluster, ApplicationId application) { - int requestedNodes = capacity.nodeCount(); - + public int decideSize(int requested, Capacity capacity, ClusterSpec cluster, ApplicationId application) { if (application.instance().isTester()) return 1; - ensureRedundancy(requestedNodes, cluster, capacity.canFail()); - - if (capacity.isRequired()) return requestedNodes; - + ensureRedundancy(requested, cluster, capacity.canFail()); + if (capacity.isRequired()) return requested; switch(zone.environment()) { case dev : case test : return 1; - case perf : return Math.min(capacity.nodeCount(), 3); - case staging: return requestedNodes <= 1 ? requestedNodes : Math.max(2, requestedNodes / 10); - case prod : return requestedNodes; + case perf : return Math.min(requested, 3); + case staging: return requested <= 1 ? requested : Math.max(2, requested / 10); + case prod : return requested; default : throw new IllegalArgumentException("Unsupported environment " + zone.environment()); } } - public NodeResources decideNodeResources(Capacity capacity, ClusterSpec cluster) { - NodeResources resources = capacity.nodeResources().orElse(defaultNodeResources(cluster.type())); - ensureSufficientResources(resources, cluster); + public NodeResources decideNodeResources(NodeResources requested, Capacity capacity, ClusterSpec cluster) { + if (requested == NodeResources.unspecified) + requested = defaultNodeResources(cluster.type()); + ensureSufficientResources(requested, cluster); - if (capacity.isRequired()) return resources; + if (capacity.isRequired()) return requested; // Allow slow storage in zones which are not performance sensitive if (zone.system().isCd() || zone.environment() == Environment.dev || zone.environment() == Environment.test) - resources = resources.with(NodeResources.DiskSpeed.any).with(NodeResources.StorageType.any); + requested = requested.with(NodeResources.DiskSpeed.any).with(NodeResources.StorageType.any); // Dev does not cap the cpu of containers since usage is spotty: Allocate just a small amount exclusively // Do not cap in AWS as hosts are allocated on demand and 1-to-1, so the node can use the entire host if (zone.environment() == Environment.dev && !zone.region().value().contains("aws-")) - resources = resources.withVcpu(0.1); + requested = requested.withVcpu(0.1); - return resources; + return requested; } private void ensureSufficientResources(NodeResources resources, ClusterSpec cluster) { double minMemoryGb = nodeResourceLimits.minMemoryGb(cluster.type()); if (resources.memoryGb() >= minMemoryGb) return; - throw new IllegalArgumentException(String.format(Locale.ENGLISH, "Must specify at least %.2f Gb of memory for %s cluster '%s', was: %.2f Gb", minMemoryGb, cluster.type().name(), cluster.id().value(), resources.memoryGb())); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java index 4beab8a3e80..9edcfd6c697 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java @@ -16,7 +16,7 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Logger; /** - * Multithread safe class to get and set docker images for given host types. + * Multithread safe class to get and set docker images for given node types. * * @author freva */ @@ -57,9 +57,11 @@ public class DockerImages { return dockerImages.get(); } - /** Returns the current docker image for given node type, or default */ + /** Returns the current docker image for given node type, or the type for corresponding child nodes + * if it is a Docker host, or default */ public DockerImage dockerImageFor(NodeType type) { - return getDockerImages().getOrDefault(type, defaultImage); + NodeType typeToUseForLookup = type.isDockerHost() ? type.childNodeType() : type; + return getDockerImages().getOrDefault(typeToUseForLookup, defaultImage); } /** Set the docker image for nodes of given type */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index 1086a3a7cd9..fb83385920c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -86,12 +86,10 @@ public class InfraDeployerImpl implements InfraDeployer { try (Mutex lock = nodeRepository.lock(application.getApplicationId())) { NodeType nodeType = application.getCapacity().type(); Version targetVersion = infrastructureVersions.getTargetVersionFor(nodeType); - hostSpecs = provisioner.prepare( - application.getApplicationId(), - application.getClusterSpecWithVersion(targetVersion), - application.getCapacity(), - 1, // groups - logger::log); + hostSpecs = provisioner.prepare(application.getApplicationId(), + application.getClusterSpecWithVersion(targetVersion), + application.getCapacity(), + logger::log); prepared = true; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 73061acd9c1..d03aa0cac91 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; @@ -15,10 +16,14 @@ import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; import com.yahoo.log.LogLevel; +import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; @@ -70,46 +75,53 @@ public class NodeRepositoryProvisioner implements Provisioner { this.activator = new Activator(nodeRepository, loadBalancerProvisioner); } + /** * Returns a list of nodes in the prepared or active state, matching the given constraints. * The nodes are ordered by increasing index number. */ + @Deprecated // TODO: Remove after April 2020 @Override - public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requestedCapacity, + public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requestedCapacity, int wantedGroups, ProvisionLogger logger) { - if (cluster.group().isPresent()) throw new IllegalArgumentException("Node requests cannot specify a group"); - if (requestedCapacity.nodeCount() > 0 && requestedCapacity.nodeCount() % wantedGroups != 0) - throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " + wantedGroups + " groups, " + - "which doesn't allow the nodes to be divided evenly into groups"); + return prepare(application, cluster, requestedCapacity.withGroups(wantedGroups), logger); + } + /** + * Returns a list of nodes in the prepared or active state, matching the given constraints. + * The nodes are ordered by increasing index number. + */ + @Override + public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requested, + ProvisionLogger logger) { log.log(zone.system().isCd() ? Level.INFO : LogLevel.DEBUG, - () -> "Received deploy prepare request for " + requestedCapacity + " in " + - wantedGroups + " groups for application " + application + ", cluster " + cluster); - - int effectiveGroups; - NodeSpec requestedNodes; - Optional<NodeResources> resources = requestedCapacity.nodeResources(); - if ( requestedCapacity.type() == NodeType.tenant) { - int nodeCount = capacityPolicies.decideSize(requestedCapacity, cluster, application); - if (zone.environment().isManuallyDeployed() && nodeCount < requestedCapacity.nodeCount()) - logger.log(Level.INFO, "Requested " + requestedCapacity.nodeCount() + " nodes for " + cluster + - ", downscaling to " + nodeCount + " nodes in " + zone.environment()); - resources = Optional.of(capacityPolicies.decideNodeResources(requestedCapacity, cluster)); - boolean exclusive = capacityPolicies.decideExclusivity(cluster.isExclusive()); - effectiveGroups = Math.min(wantedGroups, nodeCount); // cannot have more groups than nodes - requestedNodes = NodeSpec.from(nodeCount, resources.get(), exclusive, requestedCapacity.canFail()); + () -> "Received deploy prepare request for " + requested + + " for application " + application + ", cluster " + cluster); + + if (cluster.group().isPresent()) throw new IllegalArgumentException("Node requests cannot specify a group"); - if ( ! hasQuota(application, nodeCount)) - throw new IllegalArgumentException(requestedCapacity + " requested for " + cluster + - (requestedCapacity.nodeCount() != nodeCount ? " resolved to " + nodeCount + " nodes" : "") + - " exceeds your quota. Resolve this at https://cloud.vespa.ai/quota"); + if ( ! hasQuota(application, requested.maxResources().nodes())) + throw new IllegalArgumentException(requested + " requested for " + cluster + + ". Max value exceeds your quota. Resolve this at https://cloud.vespa.ai/quota"); + + int groups; + NodeResources resources; + NodeSpec nodeSpec; + if ( requested.type() == NodeType.tenant) { + ClusterResources target = decideTargetResources(application, cluster.id(), requested); + int nodeCount = capacityPolicies.decideSize(target.nodes(), requested, cluster, application); + resources = capacityPolicies.decideNodeResources(target.nodeResources(), requested, cluster); + boolean exclusive = capacityPolicies.decideExclusivity(cluster.isExclusive()); + groups = Math.min(target.groups(), nodeCount); // cannot have more groups than nodes + nodeSpec = NodeSpec.from(nodeCount, resources, exclusive, requested.canFail()); + logIfDownscaled(target.nodes(), nodeCount, cluster, logger); } else { - requestedNodes = NodeSpec.from(requestedCapacity.type()); - effectiveGroups = 1; // type request with multiple groups is not supported + groups = 1; // type request with multiple groups is not supported + resources = requested.minResources().nodeResources(); + nodeSpec = NodeSpec.from(requested.type()); } - - return asSortedHosts(preparer.prepare(application, cluster, requestedNodes, effectiveGroups), resources); + return asSortedHosts(preparer.prepare(application, cluster, nodeSpec, groups), resources); } @Override @@ -129,6 +141,40 @@ public class NodeRepositoryProvisioner implements Provisioner { loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.deactivate(application, transaction)); } + /** + * Returns the target cluster resources, a value between the min and max in the requested capacity, + * and updates the application store with the received min and max, + */ + private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec.Id clusterId, Capacity requested) { + try (Mutex lock = nodeRepository.lock(applicationId)) { + Application application = nodeRepository.applications().get(applicationId, true); + application = application.withClusterLimits(clusterId, requested.minResources(), requested.maxResources()); + nodeRepository.applications().set(applicationId, application, lock); + return application.cluster(clusterId).targetResources() + .orElseGet(() -> currentResources(applicationId, clusterId, requested) + .orElse(requested.minResources())); + } + } + + /** Returns the current resources of this cluster, if it's already deployed and inside the requested limits */ + private Optional<ClusterResources> currentResources(ApplicationId applicationId, + ClusterSpec.Id clusterId, + Capacity requested) { + List<Node> nodes = NodeList.copyOf(nodeRepository.getNodes(applicationId, Node.State.active)) + .cluster(clusterId).not().retired().asList(); + if (nodes.size() < 1) return Optional.empty(); + long groups = nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); + var resources = new ClusterResources(nodes.size(), (int)groups, nodes.get(0).flavor().resources()); + if ( ! resources.isWithin(requested.minResources(), requested.maxResources())) return Optional.empty(); + return Optional.of(resources); + } + + private void logIfDownscaled(int targetNodes, int actualNodes, ClusterSpec cluster, ProvisionLogger logger) { + if (zone.environment().isManuallyDeployed() && actualNodes < targetNodes) + logger.log(Level.INFO, "Requested " + targetNodes + " nodes for " + cluster + + ", downscaling to " + actualNodes + " nodes in " + zone.environment()); + } + private boolean hasQuota(ApplicationId application, int requestedNodes) { if ( ! this.zone.system().isPublic()) return true; // no quota management @@ -137,7 +183,7 @@ public class NodeRepositoryProvisioner implements Provisioner { return requestedNodes <= 5; } - private List<HostSpec> asSortedHosts(List<Node> nodes, Optional<NodeResources> requestedResources) { + private List<HostSpec> asSortedHosts(List<Node> nodes, NodeResources requestedResources) { nodes.sort(Comparator.comparingInt(node -> node.allocation().get().membership().index())); List<HostSpec> hosts = new ArrayList<>(nodes.size()); for (Node node : nodes) { @@ -149,7 +195,7 @@ public class NodeRepositoryProvisioner implements Provisioner { Optional.of(nodeAllocation.membership()), node.status().vespaVersion(), nodeAllocation.networkPorts(), - requestedResources, + requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources), node.status().dockerImage().map(DockerImage::repository))); if (nodeAllocation.networkPorts().isPresent()) { log.log(LogLevel.DEBUG, () -> "Prepared node " + node.hostname() + " has port allocations"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 248cfbec662..a4412a502aa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -219,7 +219,7 @@ class NodesResponse extends HttpResponse { object.setString("storageType", serializer.toString(resources.storageType())); } - // Hack: For non-docker noder, return current docker image as default prefix + current Vespa version + // Hack: For non-docker nodes, return current docker image as default prefix + current Vespa version // TODO: Remove current + wanted docker image from response for non-docker types private Optional<DockerImage> currentDockerImage(Node node) { return node.status().dockerImage() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java index 7ab42093bab..5ec5c2c08e8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java @@ -146,8 +146,8 @@ public class MockDeployer implements Deployer { this.clusterContexts = clusterContexts; } - public ApplicationContext(ApplicationId id, ClusterSpec cluster, Capacity capacity, int groups) { - this(id, List.of(new ClusterContext(id, cluster, capacity, groups))); + public ApplicationContext(ApplicationId id, ClusterSpec cluster, Capacity capacity) { + this(id, List.of(new ClusterContext(id, cluster, capacity))); } public ApplicationId id() { return id; } @@ -169,13 +169,11 @@ public class MockDeployer implements Deployer { private final ApplicationId id; private final ClusterSpec cluster; private final Capacity capacity; - private final int groups; - public ClusterContext(ApplicationId id, ClusterSpec cluster, Capacity capacity, int groups) { + public ClusterContext(ApplicationId id, ClusterSpec cluster, Capacity capacity) { this.id = id; this.cluster = cluster; this.capacity = capacity; - this.groups = groups; } public ApplicationId id() { return id; } @@ -183,7 +181,7 @@ public class MockDeployer implements Deployer { public ClusterSpec cluster() { return cluster; } private List<HostSpec> prepare(NodeRepositoryProvisioner provisioner) { - return provisioner.prepare(id, cluster, capacity, groups, null); + return provisioner.prepare(id, cluster, capacity, null); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 4f72864b087..8ff17a8e5a3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; @@ -139,19 +140,19 @@ public class MockNodeRepository extends NodeRepository { ApplicationId zoneApp = ApplicationId.from(TenantName.from("zoneapp"), ApplicationName.from("zoneapp"), InstanceName.from("zoneapp")); ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build(); - activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), 1, null), zoneApp, provisioner); + activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), null), zoneApp, provisioner); ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")).vespaVersion("6.42").build(); - provisioner.prepare(app1, cluster1, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null); + provisioner.prepare(app1, cluster1, Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1))), null); ApplicationId app2 = ApplicationId.from(TenantName.from("tenant2"), ApplicationName.from("application2"), InstanceName.from("instance2")); ClusterSpec cluster2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id2")).vespaVersion("6.42").build(); - activate(provisioner.prepare(app2, cluster2, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null), app2, provisioner); + activate(provisioner.prepare(app2, cluster2, Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1))), null), app2, provisioner); ApplicationId app3 = ApplicationId.from(TenantName.from("tenant3"), ApplicationName.from("application3"), InstanceName.from("instance3")); ClusterSpec cluster3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id3")).vespaVersion("6.42").build(); - activate(provisioner.prepare(app3, cluster3, Capacity.fromCount(2, new NodeResources(1, 4, 100, 1), false, true), 1, null), app3, provisioner); + activate(provisioner.prepare(app3, cluster3, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 100, 1)), false, true), null), app3, provisioner); List<Node> largeNodes = new ArrayList<>(); largeNodes.add(createNode("node13", "host13.yahoo.com", ipConfig(13), Optional.empty(), @@ -162,7 +163,7 @@ public class MockNodeRepository extends NodeRepository { setReady(largeNodes, Agent.system, getClass().getSimpleName()); ApplicationId app4 = ApplicationId.from(TenantName.from("tenant4"), ApplicationName.from("application4"), InstanceName.from("instance4")); ClusterSpec cluster4 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id4")).vespaVersion("6.42").build(); - activate(provisioner.prepare(app4, cluster4, Capacity.fromCount(2, new NodeResources(10, 48, 500, 1), false, true), 1, null), app4, provisioner); + activate(provisioner.prepare(app4, cluster4, Capacity.from(new ClusterResources(2, 1, new NodeResources(10, 48, 500, 1)), false, true), null), app4, provisioner); } private void activate(List<HostSpec> hosts, ApplicationId application, NodeRepositoryProvisioner provisioner) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java index 3915ae41d6e..045d0cad1ad 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java @@ -20,11 +20,17 @@ import java.util.List; public class MockProvisioner implements Provisioner { @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { return Collections.emptyList(); } @Override + public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + return Collections.emptyList(); + } + + @Override public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java index d154af4f025..f02acdc1fca 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -2,12 +2,15 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import org.junit.Test; @@ -47,7 +50,13 @@ public class AutoscalingIntegrationTest { tester.nodeMetricsDb().gc(tester.clock()); } - var scaledResources = autoscaler.autoscale(tester.nodeRepository().getNodes(application1)); + ClusterResources min = new ClusterResources(2, 1, nodes); + ClusterResources max = new ClusterResources(2, 1, nodes); + + Application application = tester.nodeRepository().applications().get(application1, true).withClusterLimits(cluster1.id(), min, max); + tester.nodeRepository().applications().set(application1, application, tester.nodeRepository().lock(application1)); + var scaledResources = autoscaler.autoscale(application.cluster(cluster1.id()), + tester.nodeRepository().getNodes(application1)); assertTrue(scaledResources.isPresent()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 39259bf44f8..8bfb17c0bd4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.google.common.collect.Sets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -31,6 +32,8 @@ public class AutoscalingTest { @Test public void testAutoscalingSingleContentGroup() { NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); AutoscalingTester tester = new AutoscalingTester(resources); ApplicationId application1 = tester.applicationId("application1"); @@ -39,37 +42,39 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 5, 1, resources); - assertTrue("No measurements -> No change", tester.autoscale(application1).isEmpty()); + assertTrue("No measurements -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); tester.addMeasurements(Resource.cpu, 0.25f, 1f, 60, application1); - assertTrue("Too few measurements -> No change", tester.autoscale(application1).isEmpty()); + assertTrue("Too few measurements -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); tester.addMeasurements(Resource.cpu, 0.25f, 1f, 60, application1); AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", 15, 1, 1.3, 28.6, 28.6, - tester.autoscale(application1)); + tester.autoscale(application1, cluster1.id(), min, max)); tester.deploy(application1, cluster1, scaledResources); - assertTrue("Cluster in flux -> No further change", tester.autoscale(application1).isEmpty()); + assertTrue("Cluster in flux -> No further change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); tester.deactivateRetired(application1, cluster1, scaledResources); tester.addMeasurements(Resource.cpu, 0.8f, 1f, 3, application1); assertTrue("Load change is large, but insufficient measurements for new config -> No change", - tester.autoscale(application1).isEmpty()); + tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); tester.addMeasurements(Resource.cpu, 0.19f, 1f, 100, application1); - assertEquals("Load change is small -> No change", Optional.empty(), tester.autoscale(application1)); + assertEquals("Load change is small -> No change", Optional.empty(), tester.autoscale(application1, cluster1.id(), min, max)); tester.addMeasurements(Resource.cpu, 0.1f, 1f, 120, application1); - tester.assertResources("Scaling down since resource usage has gone down significantly", - 26, 1, 0.6, 16.0, 16.0, - tester.autoscale(application1)); + tester.assertResources("Scaling down to minimum since usage has gone down significantly", + 14, 1, 1.0, 30.8, 30.8, + tester.autoscale(application1, cluster1.id(), min, max)); } /** We prefer fewer nodes for container clusters as (we assume) they all use the same disk and memory */ @Test public void testAutoscalingSingleContainerGroup() { NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); AutoscalingTester tester = new AutoscalingTester(resources); ApplicationId application1 = tester.applicationId("application1"); @@ -81,7 +86,7 @@ public class AutoscalingTest { tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since cpu usage is too high", 7, 1, 2.6, 80.0, 80.0, - tester.autoscale(application1)); + tester.autoscale(application1, cluster1.id(), min, max)); tester.deploy(application1, cluster1, scaledResources); tester.deactivateRetired(application1, cluster1, scaledResources); @@ -89,12 +94,92 @@ public class AutoscalingTest { tester.addMeasurements(Resource.cpu, 0.1f, 1f, 120, application1); tester.assertResources("Scaling down since cpu usage has gone down", 4, 1, 2.4, 68.6, 68.6, - tester.autoscale(application1)); + tester.autoscale(application1, cluster1.id(), min, max)); + } + + @Test + public void testAutoscalingRespectsUpperLimit() { + NodeResources hostResources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); + AutoscalingTester tester = new AutoscalingTester(hostResources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, + new NodeResources(1.9, 70, 70, 1)); + tester.addMeasurements(Resource.cpu, 0.25f, 120, application1); + tester.addMeasurements(Resource.memory, 0.95f, 120, application1); + tester.addMeasurements(Resource.disk, 0.95f, 120, application1); + tester.assertResources("Scaling up to limit since resource usage is too high", + 6, 1, 2.4, 78.0, 79.0, + tester.autoscale(application1, cluster1.id(), min, max)); + } + + @Test + public void testAutoscalingRespectsLowerLimit() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 4, 1, new NodeResources(1.8, 7.4, 8.5, 1)); + ClusterResources max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, resources); + tester.addMeasurements(Resource.cpu, 0.05f, 120, application1); + tester.addMeasurements(Resource.memory, 0.05f, 120, application1); + tester.addMeasurements(Resource.disk, 0.05f, 120, application1); + tester.assertResources("Scaling down to limit since resource usage is low", + 4, 1, 1.8, 7.4, 8.5, + tester.autoscale(application1, cluster1.id(), min, max)); + } + + @Test + public void testAutoscalingRespectsGroupLimit() { + NodeResources hostResources = new NodeResources(30.0, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(18, 6, new NodeResources(100, 1000, 1000, 1)); + AutoscalingTester tester = new AutoscalingTester(hostResources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 5, new NodeResources(3.0, 10, 10, 1)); + tester.addMeasurements(Resource.cpu, 0.3f, 1f, 240, application1); + tester.assertResources("Scaling up since resource usage is too high", + 6, 6, 3.6, 8.0, 8.0, + tester.autoscale(application1, cluster1.id(), min, max)); + } + + /** This condition ensures we get recommendation suggestions when deactivated */ + @Test + public void testAutoscalingLimitsAreIgnoredIfMinEqualsMax() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = min; + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, resources); + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); + tester.assertResources("Scaling up since resource usage is too high", + 7, 1, 2.6, 80.0, 80.0, + tester.autoscale(application1, cluster1.id(), min, max)); } @Test public void testAutoscalingGroupSize1() { NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(20, 20, new NodeResources(100, 1000, 1000, 1)); AutoscalingTester tester = new AutoscalingTester(resources); ApplicationId application1 = tester.applicationId("application1"); @@ -105,12 +190,14 @@ public class AutoscalingTest { tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); tester.assertResources("Scaling up since resource usage is too high", 7, 7, 2.5, 80.0, 80.0, - tester.autoscale(application1)); + tester.autoscale(application1, cluster1.id(), min, max)); } @Test public void testAutoscalingGroupSize3() { NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1)); AutoscalingTester tester = new AutoscalingTester(resources); ApplicationId application1 = tester.applicationId("application1"); @@ -121,12 +208,14 @@ public class AutoscalingTest { tester.addMeasurements(Resource.cpu, 0.22f, 1f, 120, application1); tester.assertResources("Scaling up since resource usage is too high", 9, 3, 2.7, 83.3, 83.3, - tester.autoscale(application1)); + tester.autoscale(application1, cluster1.id(), min, max)); } @Test public void testAutoscalingAvoidsIllegalConfigurations() { NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); AutoscalingTester tester = new AutoscalingTester(resources); ApplicationId application1 = tester.applicationId("application1"); @@ -137,11 +226,13 @@ public class AutoscalingTest { tester.addMeasurements(Resource.memory, 0.02f, 1f, 120, application1); tester.assertResources("Scaling down", 6, 1, 3.0, 4.0, 100.0, - tester.autoscale(application1)); + tester.autoscale(application1, cluster1.id(), min, max)); } @Test public void testAutoscalingAws() { + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); List<Flavor> flavors = new ArrayList<>(); flavors.add(new Flavor("aws-xlarge", new NodeResources(3, 200, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); flavors.add(new Flavor("aws-large", new NodeResources(3, 150, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); @@ -160,7 +251,7 @@ public class AutoscalingTest { tester.addMeasurements(Resource.memory, 0.9f, 0.6f, 120, application1); AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high.", 8, 1, 3, 83, 34.3, - tester.autoscale(application1)); + tester.autoscale(application1, cluster1.id(), min, max)); tester.deploy(application1, cluster1, scaledResources); tester.deactivateRetired(application1, cluster1, scaledResources); @@ -168,7 +259,7 @@ public class AutoscalingTest { tester.addMeasurements(Resource.memory, 0.3f, 0.6f, 1000, application1); tester.assertResources("Scaling down since resource usage has gone down", 5, 1, 3, 83, 36, - tester.autoscale(application1)); + tester.autoscale(application1, cluster1.id(), min, max)); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 3ba230f9ccd..ebc4d158ded 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -19,6 +20,7 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; @@ -82,7 +84,7 @@ class AutoscalingTester { } public List<HostSpec> deploy(ApplicationId application, ClusterSpec cluster, int nodes, int groups, NodeResources resources) { - List<HostSpec> hosts = provisioningTester.prepare(application, cluster, Capacity.fromCount(nodes, resources), groups); + List<HostSpec> hosts = provisioningTester.prepare(application, cluster, Capacity.from(new ClusterResources(nodes, groups, resources))); for (HostSpec host : hosts) makeReady(host.hostname()); provisioningTester.deployZoneApp(); @@ -139,8 +141,25 @@ class AutoscalingTester { } } - public Optional<AllocatableClusterResources> autoscale(ApplicationId application) { - return autoscaler.autoscale(nodeRepository().getNodes(application, Node.State.active)); + public void addMeasurements(Resource resource, float value, int count, ApplicationId applicationId) { + List<Node> nodes = nodeRepository().getNodes(applicationId, Node.State.active); + for (int i = 0; i < count; i++) { + clock().advance(Duration.ofMinutes(1)); + for (Node node : nodes) { + db.add(List.of(new NodeMetrics.MetricValue(node.hostname(), + resource.metricName(), + clock().instant().toEpochMilli(), + value * 100))); // the metrics are in % + } + } + } + + public Optional<AllocatableClusterResources> autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, + ClusterResources min, ClusterResources max) { + Application application = nodeRepository().applications().get(applicationId, true).withClusterLimits(clusterId, min, max); + nodeRepository().applications().set(applicationId, application, nodeRepository().lock(applicationId)); + return autoscaler.autoscale(application.cluster(clusterId), + nodeRepository().getNodes(applicationId, Node.State.active)); } public AllocatableClusterResources assertResources(String message, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java index 8c1a27f2fd1..6bf52218302 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java @@ -3,7 +3,10 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.NodeResources; +import com.yahoo.vdslib.state.NodeState; +import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import com.yahoo.vespa.applicationmodel.HostName; @@ -13,6 +16,7 @@ import java.util.ArrayList; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class NodeMetricsFetcherTest { @@ -29,8 +33,8 @@ public class NodeMetricsFetcherTest { ApplicationId application1 = tester.makeApplicationId(); ApplicationId application2 = tester.makeApplicationId(); - tester.deploy(application1, Capacity.fromCount(2, resources)); // host-1.yahoo.com, host-2.yahoo.com - tester.deploy(application2, Capacity.fromCount(2, resources)); // host-4.yahoo.com, host-3.yahoo.com + tester.deploy(application1, Capacity.from(new ClusterResources(2, 1, resources))); // host-1.yahoo.com, host-2.yahoo.com + tester.deploy(application2, Capacity.from(new ClusterResources(2, 1, resources))); // host-4.yahoo.com, host-3.yahoo.com orchestrator.suspend(new HostName("host-4.yahoo.com")); @@ -57,6 +61,12 @@ public class NodeMetricsFetcherTest { assertEquals("metric value mem_total.util: 15.0 at 1970-01-01T00:21:40Z for host-3.yahoo.com", values.get(1).toString()); assertEquals("metric value disk.util: 20.0 at 1970-01-01T00:21:40Z for host-3.yahoo.com", values.get(2).toString()); } + + { + tester.nodeRepository().write(tester.nodeRepository().getNodes(application1, Node.State.active).get(0).retire(tester.clock().instant()), + tester.nodeRepository().lock(application1)); + assertTrue("No metrics fetching while unstable", fetcher.fetchMetrics(application1).isEmpty()); + } } private static class MockHttpClient implements NodeMetricsFetcher.HttpClient { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java new file mode 100644 index 00000000000..da169cba08f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -0,0 +1,113 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.vespa.hosted.provision.autoscale.Resource; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertTrue; + +/** + * Tests the autoscaling maintainer integration. + * The specific recommendations of the autoscaler are not tested here. + * + * @author bratseth + */ +public class AutoscalingMaintainerTest { + + @Test + public void testAutoscalingMaintainer() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east3"))).flavorsConfig(flavorsConfig()).build(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = tester.clusterSpec(); + + ApplicationId app2 = tester.makeApplicationId("app2"); + ClusterSpec cluster2 = tester.clusterSpec(); + + NodeResources lowResources = new NodeResources(4, 4, 10, 0.1); + NodeResources highResources = new NodeResources(6.5, 9, 20, 0.1); + + Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( + app1, new MockDeployer.ApplicationContext(app1, cluster1, Capacity.from(new ClusterResources(2, 1, lowResources))), + app2, new MockDeployer.ApplicationContext(app2, cluster2, Capacity.from(new ClusterResources(2, 1, highResources)))); + MockDeployer deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps); + + NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(); + AutoscalingMaintainer maintainer = new AutoscalingMaintainer(tester.nodeRepository(), + tester.identityHostResourcesCalculator(), + nodeMetricsDb, + deployer, + Duration.ofMinutes(1)); + maintainer.maintain(); // noop + assertTrue(deployer.lastDeployTime(app1).isEmpty()); + assertTrue(deployer.lastDeployTime(app2).isEmpty()); + + tester.makeReadyNodes(20, "flt", NodeType.host, 8); + tester.deployZoneApp(); + + tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + false, true)); + tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + new ClusterResources(10, 1, new NodeResources(6.5, 9, 20, 0.1)), + false, true)); + + maintainer.maintain(); // noop + assertTrue(deployer.lastDeployTime(app1).isEmpty()); + assertTrue(deployer.lastDeployTime(app2).isEmpty()); + + addMeasurements(Resource.cpu, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.memory, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.disk, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.cpu, 0.9f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.memory, 0.9f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.disk, 0.9f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + + maintainer.maintain(); + assertTrue(deployer.lastDeployTime(app1).isEmpty()); // since autoscaling is off + assertTrue(deployer.lastDeployTime(app2).isPresent()); + } + + public void addMeasurements(Resource resource, float value, int count, ApplicationId applicationId, + NodeRepository nodeRepository, NodeMetricsDb db) { + List<Node> nodes = nodeRepository.getNodes(applicationId, Node.State.active); + for (int i = 0; i < count; i++) { + for (Node node : nodes) + db.add(List.of(new NodeMetrics.MetricValue(node.hostname(), + resource.metricName(), + nodeRepository.clock().instant().toEpochMilli(), + value * 100))); // the metrics are in % + } + } + + private FlavorsConfig flavorsConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("flt", 30, 30, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("cpu", 40, 20, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("mem", 20, 40, 40, 3, Flavor.Type.BARE_METAL); + return b.build(); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index e8b9813a8a2..17521261e1b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; @@ -322,12 +323,12 @@ public class FailedExpirerTest { public FailureScenario allocate(ClusterSpec.Type clusterType, NodeResources flavor, String... hostname) { ClusterSpec clusterSpec = ClusterSpec.request(clusterType, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); - Capacity capacity = Capacity.fromCount(hostname.length, Optional.of(flavor), false, true); + Capacity capacity = Capacity.from(new ClusterResources(hostname.length, 1, flavor), false, true); return allocate(applicationId, clusterSpec, capacity); } public FailureScenario allocate(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity capacity) { - List<HostSpec> preparedNodes = provisioner.prepare(applicationId, clusterSpec, capacity, 1, null); + List<HostSpec> preparedNodes = provisioner.prepare(applicationId, clusterSpec, capacity, null); NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); provisioner.activate(transaction, applicationId, Set.copyOf(preparedNodes)); transaction.commit(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index 17d8b702269..9bb3a55abfd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostSpec; @@ -54,7 +55,7 @@ public class InactiveAndFailedExpirerTest { // Allocate then deallocate 2 nodes ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); - List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1); + List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources))); tester.activate(applicationId, new HashSet<>(preparedNodes)); assertEquals(2, tester.getNodes(applicationId, Node.State.active).size()); tester.deactivate(applicationId); @@ -92,7 +93,7 @@ public class InactiveAndFailedExpirerTest { // Allocate and deallocate a single node ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); - List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1); + List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources))); tester.activate(applicationId, new HashSet<>(preparedNodes)); assertEquals(2, tester.getNodes(applicationId, Node.State.active).size()); tester.deactivate(applicationId); @@ -124,8 +125,7 @@ public class InactiveAndFailedExpirerTest { { List<HostSpec> hostSpecs = tester.prepare(applicationId, cluster, - Capacity.fromCount(2, nodeResources), - 1); + Capacity.from(new ClusterResources(2, 1, nodeResources))); tester.activate(applicationId, new HashSet<>(hostSpecs)); assertEquals(2, tester.getNodes(applicationId, Node.State.active).size()); } @@ -134,7 +134,7 @@ public class InactiveAndFailedExpirerTest { { Node toRetire = tester.getNodes(applicationId, Node.State.active).asList().get(0); tester.patchNode(toRetire.withWantToRetire(true, Agent.operator, tester.clock().instant())); - List<HostSpec> hostSpecs = tester.prepare(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1); + List<HostSpec> hostSpecs = tester.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources))); tester.activate(applicationId, new HashSet<>(hostSpecs)); } @@ -146,10 +146,8 @@ public class InactiveAndFailedExpirerTest { Collections.singletonMap( applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, - Capacity.fromCount(2, - nodeResources, - false, true), - 1) + Capacity.from(new ClusterResources(2, 1, nodeResources), + false, true)) ) ); Orchestrator orchestrator = mock(Orchestrator.class); @@ -174,7 +172,7 @@ public class InactiveAndFailedExpirerTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); tester.makeReadyNodes(1, nodeResources); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); - List<HostSpec> preparedNodes = tester.prepare(testerId, cluster, Capacity.fromCount(2, nodeResources), 1); + List<HostSpec> preparedNodes = tester.prepare(testerId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources))); tester.activate(testerId, new HashSet<>(preparedNodes)); assertEquals(1, tester.getNodes(testerId, Node.State.active).size()); tester.deactivate(testerId); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java index 4344016c6fe..664809dc3ab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; @@ -14,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import java.time.Instant; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index aa262bdf751..ca7c33f96bd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.test.ManualClock; +import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.curator.Curator; @@ -163,6 +164,10 @@ public class MetricsReporterTest { container2 = container2.with(allocation(Optional.of("app2"), container2).get()); nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockUnallocated())); + NestedTransaction transaction = new NestedTransaction(); + nodeRepository.activate(nodeRepository.getNodes(NodeType.host), transaction); + transaction.commit(); + Orchestrator orchestrator = mock(Orchestrator.class); when(orchestrator.getHostInfo(eq(reference), any())).thenReturn(HostInfo.createNoRemarks()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 0efca6ef826..585d57aae4e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; @@ -89,17 +90,17 @@ public class NodeFailTester { // Create applications ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); - Capacity capacity1 = Capacity.fromCount(5, nodeResources, false, true); - Capacity capacity2 = Capacity.fromCount(7, nodeResources, false, true); + Capacity capacity1 = Capacity.from(new ClusterResources(5, 1, nodeResources), false, true); + Capacity capacity2 = Capacity.from(new ClusterResources(7, 1, nodeResources), false, true); tester.activate(app1, clusterApp1, capacity1); tester.activate(app2, clusterApp2, capacity2); - assertEquals(capacity1.nodeCount(), tester.nodeRepository.getNodes(app1, Node.State.active).size()); - assertEquals(capacity2.nodeCount(), tester.nodeRepository.getNodes(app2, Node.State.active).size()); + assertEquals(capacity1.minResources().nodes(), tester.nodeRepository.getNodes(app1, Node.State.active).size()); + assertEquals(capacity2.minResources().nodes(), tester.nodeRepository.getNodes(app2, Node.State.active).size()); Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( - app1, new MockDeployer.ApplicationContext(app1, clusterApp1, capacity1, 1), - app2, new MockDeployer.ApplicationContext(app2, clusterApp2, capacity2, 1)); + app1, new MockDeployer.ApplicationContext(app1, clusterApp1, capacity1), + app2, new MockDeployer.ApplicationContext(app2, clusterApp2, capacity2)); tester.deployer = new MockDeployer(tester.provisioner, tester.clock(), apps); tester.serviceMonitor = new ServiceMonitorStub(apps, tester.nodeRepository); tester.metric = new MetricsReporterTest.TestMetric(); @@ -122,20 +123,20 @@ public class NodeFailTester { ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("6.75.0").build(); ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.75.0").build(); Capacity allHosts = Capacity.fromRequiredNodeType(NodeType.host); - Capacity capacity1 = Capacity.fromCount(3, new NodeResources(1, 4, 10, 0.3), false, true); - Capacity capacity2 = Capacity.fromCount(5, new NodeResources(1, 4, 10, 0.3), false, true); + Capacity capacity1 = Capacity.from(new ClusterResources(3, 1, new NodeResources(1, 4, 10, 0.3)), false, true); + Capacity capacity2 = Capacity.from(new ClusterResources(5, 1, new NodeResources(1, 4, 10, 0.3)), false, true); tester.activate(tenantHostApp, clusterNodeAdminApp, allHosts); tester.activate(app1, clusterApp1, capacity1); tester.activate(app2, clusterApp2, capacity2); assertEquals(Set.of(tester.nodeRepository.getNodes(NodeType.host)), Set.of(tester.nodeRepository.getNodes(tenantHostApp, Node.State.active))); - assertEquals(capacity1.nodeCount(), tester.nodeRepository.getNodes(app1, Node.State.active).size()); - assertEquals(capacity2.nodeCount(), tester.nodeRepository.getNodes(app2, Node.State.active).size()); + assertEquals(capacity1.minResources().nodes(), tester.nodeRepository.getNodes(app1, Node.State.active).size()); + assertEquals(capacity2.minResources().nodes(), tester.nodeRepository.getNodes(app2, Node.State.active).size()); Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( - tenantHostApp, new MockDeployer.ApplicationContext(tenantHostApp, clusterNodeAdminApp, allHosts, 1), - app1, new MockDeployer.ApplicationContext(app1, clusterApp1, capacity1, 1), - app2, new MockDeployer.ApplicationContext(app2, clusterApp2, capacity2, 1)); + tenantHostApp, new MockDeployer.ApplicationContext(tenantHostApp, clusterNodeAdminApp, allHosts), + app1, new MockDeployer.ApplicationContext(app1, clusterApp1, capacity1), + app2, new MockDeployer.ApplicationContext(app2, clusterApp2, capacity2)); tester.deployer = new MockDeployer(tester.provisioner, tester.clock(), apps); tester.serviceMonitor = new ServiceMonitorStub(apps, tester.nodeRepository); tester.metric = new MetricsReporterTest.TestMetric(); @@ -154,7 +155,7 @@ public class NodeFailTester { assertEquals(count, tester.nodeRepository.getNodes(nodeType, Node.State.active).size()); Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( - app1, new MockDeployer.ApplicationContext(app1, clusterApp1, allNodes, 1)); + app1, new MockDeployer.ApplicationContext(app1, clusterApp1, allNodes)); tester.deployer = new MockDeployer(tester.provisioner, tester.clock(), apps); tester.serviceMonitor = new ServiceMonitorStub(apps, tester.nodeRepository); tester.metric = new MetricsReporterTest.TestMetric(); @@ -252,7 +253,7 @@ public class NodeFailTester { } private void activate(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity) { - List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, capacity, 1, null); + List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, capacity, null); NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); provisioner.activate(transaction, applicationId, hosts); transaction.commit(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index cf93e8ecf6e..2dda6c714a7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; @@ -148,9 +149,9 @@ public class OperatorChangeApplicationMaintainerTest { new InMemoryFlagSource()); Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( - app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.fromCount(wantedNodesApp1, nodeResources), 1), - app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.fromCount(wantedNodesApp2, nodeResources), 1), - app3, new MockDeployer.ApplicationContext(app3, clusterApp3, Capacity.fromRequiredNodeType(NodeType.proxy), 0)) ; + app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.from(new ClusterResources(wantedNodesApp1, 1, nodeResources))), + app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.from(new ClusterResources(wantedNodesApp2, 1, nodeResources))), + app3, new MockDeployer.ApplicationContext(app3, clusterApp3, Capacity.fromRequiredNodeType(NodeType.proxy))) ; this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index 4ca88c10bfe..3037d5972e5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.DockerImage; @@ -258,8 +259,8 @@ public class PeriodicApplicationMaintainerTest { new InMemoryFlagSource()); Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( - app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.fromCount(wantedNodesApp1, nodeResources), 1), - app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.fromCount(wantedNodesApp2, nodeResources), 1)); + app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.from(new ClusterResources(wantedNodesApp1, 1, nodeResources))), + app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.from(new ClusterResources(wantedNodesApp2, 1, nodeResources)))); this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps); this.maintainer = new TestablePeriodicApplicationMaintainer(deployer, nodeRepository, Duration.ofDays(1), // Long duration to prevent scheduled runs during test Duration.ofMinutes(30)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index b4e713ae492..387f614c5eb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -48,13 +49,13 @@ public class RebalancerTest { MetricsReporterTest.TestMetric metric = new MetricsReporterTest.TestMetric(); Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( - cpuApp, new MockDeployer.ApplicationContext(cpuApp, clusterSpec("c"), Capacity.fromCount(1, cpuResources), 1), - memApp, new MockDeployer.ApplicationContext(memApp, clusterSpec("c"), Capacity.fromCount(1, memResources), 1)); + cpuApp, new MockDeployer.ApplicationContext(cpuApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, cpuResources))), + memApp, new MockDeployer.ApplicationContext(memApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, memResources)))); MockDeployer deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps); Rebalancer rebalancer = new Rebalancer(deployer, tester.nodeRepository(), - new IdentityHostResourcesCalculator(), + tester.identityHostResourcesCalculator(), Optional.empty(), metric, tester.clock(), @@ -148,18 +149,4 @@ public class RebalancerTest { return b.build(); } - private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { - - @Override - public NodeResources realResourcesOf(Node node) { - return node.flavor().resources(); - } - - @Override - public NodeResources advertisedResourcesOf(Flavor flavor) { - return flavor.resources(); - } - - } - } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index b4f55c437cd..0fd967cad1b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; @@ -61,7 +62,7 @@ public class ReservationExpirerTest { nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("bar").instanceName("fuz").build(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); - provisioner.prepare(applicationId, cluster, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null); + provisioner.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1))), null); assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.reserved).size()); // Reservation times out diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 78600d360c9..276b9484ad4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.DockerImage; @@ -98,7 +99,9 @@ public class RetiredExpirerTest { MockDeployer deployer = new MockDeployer(provisioner, clock, - Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromCount(wantedNodes, nodeResources), 1))); + Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, + cluster, + Capacity.from(new ClusterResources(wantedNodes, 1, nodeResources))))); createRetiredExpirer(deployer).run(); assertEquals(3, nodeRepository.getNodes(applicationId, Node.State.active).size()); assertEquals(4, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); @@ -127,7 +130,9 @@ public class RetiredExpirerTest { MockDeployer deployer = new MockDeployer(provisioner, clock, - Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1))); + Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, + cluster, + Capacity.from(new ClusterResources(2, 1, nodeResources))))); createRetiredExpirer(deployer).run(); assertEquals(2, nodeRepository.getNodes(applicationId, Node.State.active).size()); assertEquals(6, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); @@ -161,7 +166,9 @@ public class RetiredExpirerTest { clock, Collections.singletonMap( applicationId, - new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromCount(wantedNodes, nodeResources), 1))); + new MockDeployer.ApplicationContext(applicationId, + cluster, + Capacity.from(new ClusterResources(wantedNodes, 1, nodeResources))))); // Allow the 1st and 3rd retired nodes permission to inactivate doNothing() @@ -197,7 +204,7 @@ public class RetiredExpirerTest { } private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodes, int groups, NodeRepositoryProvisioner provisioner) { - List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromCount(nodes, nodeResources), groups, null); + List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.from(new ClusterResources(nodes, groups, nodeResources)), null); NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); provisioner.activate(transaction, applicationId, hosts); transaction.commit(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index 1d028f13340..bbba87f443e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; @@ -45,7 +46,7 @@ public class AclProvisioningTest { // Allocate 2 nodes ApplicationId application = tester.makeApplicationId(); - List<Node> activeNodes = tester.deploy(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 1), false, true)); + List<Node> activeNodes = tester.deploy(application, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 10, 1)), false, true)); assertEquals(2, activeNodes.size()); // Get trusted nodes for the first active node @@ -210,7 +211,7 @@ public class AclProvisioningTest { } private List<Node> deploy(ApplicationId application, int nodeCount) { - return tester.deploy(application, Capacity.fromCount(nodeCount, nodeResources)); + return tester.deploy(application, Capacity.from(new ClusterResources(nodeCount, 1, nodeResources))); } private static void assertAcls(List<List<Node>> expected, NodeAcl actual) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java index 2eca8998931..cd6ae587b04 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java @@ -7,6 +7,8 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Test; +import java.util.Optional; + import static org.junit.Assert.assertEquals; /** @@ -19,6 +21,9 @@ public class DockerImagesTest { var flagSource = new InMemoryFlagSource(); var tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); + var proxyImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/proxy"); + tester.nodeRepository().dockerImages().setDockerImage(NodeType.proxy, Optional.of(proxyImage)); + // Host uses tenant default image (for preload purposes) var defaultImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"); var hosts = tester.makeReadyNodes(2, "default", NodeType.host); @@ -35,6 +40,12 @@ public class DockerImagesTest { assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(node.type())); } } + + // Proxy host uses image used by child nodes (proxy nodes), which is overridden in this case (for preload purposes) + var proxyHosts = tester.makeReadyNodes(2, "default", NodeType.proxyhost); + for (var host : proxyHosts) { + assertEquals(proxyImage, tester.nodeRepository().dockerImages().dockerImageFor(host.type())); + } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index 7c315a83ddc..5079fce4418 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostSpec; @@ -95,7 +96,7 @@ public class DockerProvisioningTest { // Activate the zone-app, thereby allocating the parents List<HostSpec> hosts = tester.prepare(zoneApplication, ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("zone-app")).vespaVersion(wantedVespaVersion).build(), - Capacity.fromRequiredNodeType(NodeType.host), 1); + Capacity.fromRequiredNodeType(NodeType.host)); tester.activate(zoneApplication, hosts); // Try allocating tenants again @@ -298,8 +299,7 @@ public class DockerProvisioningTest { private void prepareAndActivate(ApplicationId application, int nodeCount, boolean exclusive, ProvisioningTester tester) { Set<HostSpec> hosts = new HashSet<>(tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer")).vespaVersion("6.39").exclusive(exclusive).build(), - Capacity.fromCount(nodeCount, Optional.of(dockerFlavor), false, true), - 1)); + Capacity.from(new ClusterResources(nodeCount, 1, dockerFlavor), false, true))); tester.activate(application, hosts); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 661f379f271..4eca0542992 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -401,11 +402,11 @@ public class DynamicDockerAllocationTest { ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); - List<HostSpec> hosts1 = tester.prepare(application, cluster, Capacity.fromCount(2, Optional.of(NodeResources.fromLegacyName("d-2-8-50")), false, true), 1); + List<HostSpec> hosts1 = tester.prepare(application, cluster, Capacity.from(new ClusterResources(2, 1, NodeResources.fromLegacyName("d-2-8-50")), false, true)); tester.activate(application, hosts1); NodeResources resources = new NodeResources(1.5, 8, 50, 0.3); - List<HostSpec> hosts2 = tester.prepare(application, cluster, Capacity.fromCount(2, resources), 1); + List<HostSpec> hosts2 = tester.prepare(application, cluster, Capacity.from(new ClusterResources(2, 1, resources))); tester.activate(application, hosts2); assertEquals(hosts1, hosts2); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 273fc6bbb2d..ad9d13355dc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.collect.Iterators; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.HostSpec; @@ -154,7 +155,7 @@ public class LoadBalancerProvisionerTest { @Test public void provision_load_balancers_with_dynamic_node_provisioning() { - var nodes = prepare(app1, Capacity.fromCount(2, new NodeResources(1, 4, 10, 0.3), false, true), + var nodes = prepare(app1, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 10, 0.3)), false, true), true, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs"))); Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers(app1).asList().get(0); @@ -172,7 +173,7 @@ public class LoadBalancerProvisionerTest { assertSame("Load balancer is deactivated", LoadBalancer.State.inactive, lb.get().state()); // Application is redeployed - nodes = prepare(app1, Capacity.fromCount(2, new NodeResources(1, 4, 10, 0.3), false, true), + nodes = prepare(app1, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 10, 0.3)), false, true), true, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs"))); assertTrue("Load balancer is reconfigured with empty reals", tester.loadBalancerService().instances().get(lb.get().id()).reals().isEmpty()); @@ -229,7 +230,7 @@ public class LoadBalancerProvisionerTest { } private Set<HostSpec> prepare(ApplicationId application, ClusterSpec... specs) { - return prepare(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 0.3), false, true), false, specs); + return prepare(application, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 10, 0.3)), false, true), false, specs); } private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, boolean dynamicDockerNodes, ClusterSpec... specs) { @@ -240,7 +241,7 @@ public class LoadBalancerProvisionerTest { } Set<HostSpec> allNodes = new LinkedHashSet<>(); for (ClusterSpec spec : specs) { - allNodes.addAll(tester.prepare(application, spec, capacity, 1, false)); + allNodes.addAll(tester.prepare(application, spec, capacity, false)); } return allNodes; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index cb5116b541e..4a75d86f530 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostSpec; @@ -20,7 +21,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import java.util.Optional; import java.util.Set; import static org.junit.Assert.assertEquals; @@ -103,8 +103,8 @@ public class MultigroupProvisioningTest { tester.makeReadyNodes(10, small); - deploy(application1, Capacity.fromCount(1, Optional.of(small), true, true), 1, tester); - deploy(application1, Capacity.fromCount(2, Optional.of(small), true, true), 2, tester); + deploy(application1, Capacity.from(new ClusterResources(1, 1, small), true, true), tester); + deploy(application1, Capacity.from(new ClusterResources(2, 2, small), true, true), tester); } @Test @@ -116,8 +116,8 @@ public class MultigroupProvisioningTest { tester.makeReadyNodes(10, small); tester.makeReadyNodes(10, large); - deploy(application1, Capacity.fromCount(1, Optional.of(small), true, true), 1, tester); - deploy(application1, Capacity.fromCount(2, Optional.of(large), true, true), 2, tester); + deploy(application1, Capacity.from(new ClusterResources(1, 1, small), true, true), tester); + deploy(application1, Capacity.from(new ClusterResources(2, 2, large), true, true), tester); } @Test @@ -139,7 +139,7 @@ public class MultigroupProvisioningTest { tester.clock(), Collections.singletonMap(application1, new MockDeployer.ApplicationContext(application1, cluster(), - Capacity.fromCount(8, Optional.of(large), false, true), 1))); + Capacity.from(new ClusterResources(8, 1, large), false, true)))); new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, tester.clock(), Duration.ofDays(30), Duration.ofHours(12)).run(); @@ -148,21 +148,21 @@ public class MultigroupProvisioningTest { } private void deploy(ApplicationId application, int nodeCount, int groupCount, NodeResources resources, ProvisioningTester tester) { - deploy(application, Capacity.fromCount(nodeCount, Optional.of(resources), false, true), groupCount, tester); + deploy(application, Capacity.from(new ClusterResources(nodeCount, groupCount, resources), false, true), tester); } private void deploy(ApplicationId application, int nodeCount, int groupCount, ProvisioningTester tester) { - deploy(application, Capacity.fromCount(nodeCount, Optional.of(large), false, true), groupCount, tester); + deploy(application, Capacity.from(new ClusterResources(nodeCount, groupCount, large), false, true), tester); } - private void deploy(ApplicationId application, Capacity capacity, int wantedGroups, ProvisioningTester tester) { - int nodeCount = capacity.nodeCount(); - NodeResources nodeResources = capacity.nodeResources().get(); + private void deploy(ApplicationId application, Capacity capacity, ProvisioningTester tester) { + int nodeCount = capacity.minResources().nodes(); + NodeResources nodeResources = capacity.minResources().nodeResources(); int previousActiveNodeCount = tester.getNodes(application, Node.State.active).resources(nodeResources).size(); - tester.activate(application, prepare(application, capacity, wantedGroups, tester)); + tester.activate(application, prepare(application, capacity, tester)); assertEquals("Superfluous nodes are retired, but no others - went from " + previousActiveNodeCount + " to " + nodeCount + " nodes", - Math.max(0, previousActiveNodeCount - capacity.nodeCount()), + Math.max(0, previousActiveNodeCount - capacity.minResources().nodes()), tester.getNodes(application, Node.State.active).retired().resources(nodeResources).size()); assertEquals("Other flavors are retired", 0, tester.getNodes(application, Node.State.active).not().retired().not().resources(nodeResources).size()); @@ -187,26 +187,27 @@ public class MultigroupProvisioningTest { ClusterSpec.Group group = node.allocation().get().membership().cluster().group().get(); nonretiredGroups.put(group, nonretiredGroups.getOrDefault(group, 0) + 1); - if (wantedGroups > 1) - assertTrue("Group indexes are always in [0, wantedGroups>", group.index() < wantedGroups); + if (capacity.minResources().groups() > 1) + assertTrue("Group indexes are always in [0, wantedGroups>", + group.index() < capacity.minResources().groups()); } assertEquals("Total nonretired nodes", nodeCount, indexes.size()); - assertEquals("Total nonretired groups", wantedGroups, nonretiredGroups.size()); + assertEquals("Total nonretired groups", capacity.minResources().groups(), nonretiredGroups.size()); for (Integer groupSize : nonretiredGroups.values()) - assertEquals("Group size", (long)nodeCount / wantedGroups, (long)groupSize); + assertEquals("Group size", (long)nodeCount / capacity.minResources().groups(), (long)groupSize); Map<ClusterSpec.Group, Integer> allGroups = new HashMap<>(); for (Node node : tester.getNodes(application, Node.State.active).resources(nodeResources)) { ClusterSpec.Group group = node.allocation().get().membership().cluster().group().get(); allGroups.put(group, nonretiredGroups.getOrDefault(group, 0) + 1); } - assertEquals("No additional groups are retained containing retired nodes", wantedGroups, allGroups.size()); + assertEquals("No additional groups are retained containing retired nodes", capacity.minResources().groups(), allGroups.size()); } private ClusterSpec cluster() { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); } - private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, int groupCount, ProvisioningTester tester) { - return new HashSet<>(tester.prepare(application, cluster(), capacity, groupCount)); + private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, ProvisioningTester tester) { + return new HashSet<>(tester.prepare(application, cluster(), capacity)); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java index f4a65244e6f..8420bdeacfe 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java @@ -93,7 +93,7 @@ public class NodeTypeProvisioningTest { tester.provisioner(), tester.clock(), Collections.singletonMap( - application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1))); + application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity))); RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10)); @@ -157,7 +157,7 @@ public class NodeTypeProvisioningTest { MockDeployer deployer = new MockDeployer(tester.provisioner(), tester.clock(), Collections.singletonMap(application, - new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1))); + new MockDeployer.ApplicationContext(application, clusterSpec, capacity))); RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, @@ -263,7 +263,7 @@ public class NodeTypeProvisioningTest { } private List<HostSpec> deployProxies(ApplicationId application, ProvisioningTester tester) { - return tester.prepare(application, clusterSpec, capacity, 1); + return tester.prepare(application, clusterSpec, capacity); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 931d87a3265..bd8be5063fd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; @@ -402,6 +403,21 @@ public class ProvisioningTest { prepare(application, 1, 2, 3, 3, defaultResources, tester); } + @Test + public void below_resource_limit() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + + ApplicationId application = tester.makeApplicationId(); + tester.makeReadyNodes(10, defaultResources); + try { + prepare(application, 2, 2, 3, 3, + new NodeResources(2, 2, 10, 2), tester); + } + catch (IllegalArgumentException e) { + assertEquals("Must specify at least 4.00 Gb of memory for container cluster 'container0', was: 2.00 Gb", e.getMessage()); + } + } + /** Dev always uses the zone default flavor */ @Test public void dev_deployment_flavor() { @@ -492,7 +508,7 @@ public class ProvisioningTest { fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("6 nodes [vcpu: 1.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 4.0 Gbps] requested for content cluster 'content0' 6.42 exceeds your quota. Resolve this at https://cloud.vespa.ai/quota", + assertEquals("6 nodes with [vcpu: 1.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 4.0 Gbps] requested for content cluster 'content0' 6.42. Max value exceeds your quota. Resolve this at https://cloud.vespa.ai/quota", e.getMessage()); } } @@ -514,7 +530,7 @@ public class ProvisioningTest { tester.makeReadyNodes(4, defaultResources); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); - tester.prepare(application, cluster, Capacity.fromCount(5, Optional.empty(), false, false), 1); + tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified), false, false)); // No exception; Success } @@ -537,8 +553,8 @@ public class ProvisioningTest { @Test public void want_to_retire_but_cannot_fail() { - Capacity capacity = Capacity.fromCount(5, Optional.of(defaultResources), false, true); - Capacity capacityFORCED = Capacity.fromCount(5, Optional.of(defaultResources), false, false); + Capacity capacity = Capacity.from(new ClusterResources(5, 1, defaultResources), false, true); + Capacity capacityFORCED = Capacity.from(new ClusterResources(5, 1, defaultResources), false, false); ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); @@ -548,14 +564,14 @@ public class ProvisioningTest { tester.makeReadyNodes(10, defaultResources); // Allocate 5 nodes ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); - tester.activate(application, tester.prepare(application, cluster, capacity, 1)); + tester.activate(application, tester.prepare(application, cluster, capacity)); assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size()); assertEquals(0, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); // Mark the nodes as want to retire tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true)))); // redeploy without allow failing - tester.activate(application, tester.prepare(application, cluster, capacityFORCED, 1)); + tester.activate(application, tester.prepare(application, cluster, capacityFORCED)); // Nodes are not retired since that is unsafe when we cannot fail assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size()); @@ -564,7 +580,7 @@ public class ProvisioningTest { tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> assertTrue(node.status().wantToRetire())); // redeploy with allowing failing - tester.activate(application, tester.prepare(application, cluster, capacity, 1)); + tester.activate(application, tester.prepare(application, cluster, capacity)); // ... old nodes are now retired assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size()); assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); @@ -718,14 +734,12 @@ public class ProvisioningTest { // Application allocates two content nodes initially, with cluster type content ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); var initialNodes = tester.activate(application, tester.prepare(application, cluster, - Capacity.fromCount(2, defaultResources, false, false), - 1)); + Capacity.from(new ClusterResources(2, 1, defaultResources), false, false))); // Application is redeployed with cluster type combined cluster = ClusterSpec.request(ClusterSpec.Type.combined, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); var newNodes = tester.activate(application, tester.prepare(application, cluster, - Capacity.fromCount(2, defaultResources, false, false), - 1)); + Capacity.from(new ClusterResources(2, 1, defaultResources), false, false))); assertEquals("Node allocation remains the same", initialNodes, newNodes); assertEquals("Cluster type is updated", @@ -735,8 +749,7 @@ public class ProvisioningTest { // Application is redeployed with cluster type content again cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); newNodes = tester.activate(application, tester.prepare(application, cluster, - Capacity.fromCount(2, defaultResources, false, false), - 1)); + Capacity.from(new ClusterResources(2, 1, defaultResources), false, false))); assertEquals("Node allocation remains the same", initialNodes, newNodes); assertEquals("Cluster type is updated", Set.of(ClusterSpec.Type.content), @@ -773,11 +786,11 @@ public class ProvisioningTest { allHosts.addAll(content0); allHosts.addAll(content1); - Function<Integer, Capacity> capacity = count -> Capacity.fromCount(count, Optional.empty(), required, true); - int expectedContainer0Size = tester.capacityPolicies().decideSize(capacity.apply(container0Size), containerCluster0, application); - int expectedContainer1Size = tester.capacityPolicies().decideSize(capacity.apply(container1Size), containerCluster1, application); - int expectedContent0Size = tester.capacityPolicies().decideSize(capacity.apply(content0Size), contentCluster0, application); - int expectedContent1Size = tester.capacityPolicies().decideSize(capacity.apply(content1Size), contentCluster1, application); + Function<Integer, Capacity> capacity = count -> Capacity.from(new ClusterResources(count, 1, NodeResources.unspecified), required, true); + int expectedContainer0Size = tester.capacityPolicies().decideSize(container0Size, capacity.apply(container0Size), containerCluster0, application); + int expectedContainer1Size = tester.capacityPolicies().decideSize(container1Size, capacity.apply(container1Size), containerCluster1, application); + int expectedContent0Size = tester.capacityPolicies().decideSize(content0Size, capacity.apply(content0Size), contentCluster0, application); + int expectedContent1Size = tester.capacityPolicies().decideSize(content1Size, capacity.apply(content1Size), contentCluster1, application); assertEquals("Hosts in each group cluster is disjunct and the total number of unretired nodes is correct", expectedContainer0Size + expectedContainer1Size + expectedContent0Size + expectedContent1Size, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index c0ab137066c..a8df47aab1a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; @@ -127,19 +128,19 @@ public class ProvisioningTester { } public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, boolean required, NodeResources resources) { - return prepare(application, cluster, Capacity.fromCount(nodeCount, Optional.ofNullable(resources), required, true), groups); + return prepare(application, cluster, Capacity.from(new ClusterResources(nodeCount, groups, resources), required, true)); } - public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, int groups) { - return prepare(application, cluster, capacity, groups, true); + public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity) { + return prepare(application, cluster, capacity, true); } - public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, int groups, boolean idempotentPrepare) { + public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, boolean idempotentPrepare) { Set<String> reservedBefore = toHostNames(nodeRepository.getNodes(application, Node.State.reserved)); Set<String> inactiveBefore = toHostNames(nodeRepository.getNodes(application, Node.State.inactive)); - List<HostSpec> hosts1 = provisioner.prepare(application, cluster, capacity, groups, provisionLogger); + List<HostSpec> hosts1 = provisioner.prepare(application, cluster, capacity, provisionLogger); if (idempotentPrepare) { // prepare twice to ensure idempotence - List<HostSpec> hosts2 = provisioner.prepare(application, cluster, capacity, groups, provisionLogger); + List<HostSpec> hosts2 = provisioner.prepare(application, cluster, capacity, provisionLogger); assertEquals(hosts1, hosts2); } Set<String> newlyActivated = toHostNames(nodeRepository.getNodes(application, Node.State.reserved)); @@ -160,7 +161,7 @@ public class ProvisioningTester { public void prepareAndActivateInfraApplication(ApplicationId application, NodeType nodeType, Version version) { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString())).vespaVersion(version).build(); Capacity capacity = Capacity.fromRequiredNodeType(nodeType); - List<HostSpec> hostSpecs = prepare(application, cluster, capacity, 1, true); + List<HostSpec> hostSpecs = prepare(application, cluster, capacity, true); activate(application, hostSpecs); } @@ -232,6 +233,10 @@ public class ProvisioningTester { InstanceName.from(UUID.randomUUID().toString())); } + public ApplicationId makeApplicationId(String applicationName) { + return ApplicationId.from("tenant", applicationName, "default"); + } + public List<Node> makeReadyNodes(int n, String flavor) { return makeReadyNodes(n, flavor, NodeType.tenant); } @@ -333,11 +338,9 @@ public class ProvisioningTester { nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); ConfigServerApplication application = new ConfigServerApplication(); - List<HostSpec> hosts = prepare( - application.getApplicationId(), - application.getClusterSpecWithVersion(configServersVersion), - application.getCapacity(), - 1); + List<HostSpec> hosts = prepare(application.getApplicationId(), + application.getClusterSpecWithVersion(configServersVersion), + application.getCapacity()); activate(application.getApplicationId(), new HashSet<>(hosts)); return nodeRepository.getNodes(application.getApplicationId(), Node.State.active); } @@ -409,9 +412,8 @@ public class ProvisioningTester { public void deployZoneApp() { ApplicationId applicationId = makeApplicationId(); List<HostSpec> list = prepare(applicationId, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build(), - Capacity.fromRequiredNodeType(NodeType.host), - 1); + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build(), + Capacity.fromRequiredNodeType(NodeType.host)); activate(applicationId, Set.copyOf(list)); } @@ -420,12 +422,15 @@ public class ProvisioningTester { } public List<Node> deploy(ApplicationId application, Capacity capacity) { - List<HostSpec> prepared = prepare(application, clusterSpec(), capacity, 1); + return deploy(application, clusterSpec(), capacity); + } + + public List<Node> deploy(ApplicationId application, ClusterSpec cluster, Capacity capacity) { + List<HostSpec> prepared = prepare(application, cluster, capacity); activate(application, Set.copyOf(prepared)); return getNodes(application, Node.State.active).asList(); } - /** Returns the hosts from the input list which are not retired */ public List<HostSpec> nonRetired(Collection<HostSpec> hosts) { return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList()); @@ -524,4 +529,22 @@ public class ProvisioningTester { @Override public void log(Level level, String message) { } } + public IdentityHostResourcesCalculator identityHostResourcesCalculator() { + return new IdentityHostResourcesCalculator(); + } + + private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { + + @Override + public NodeResources realResourcesOf(Node node) { + return node.flavor().resources(); + } + + @Override + public NodeResources advertisedResourcesOf(Flavor flavor) { + return flavor.resources(); + } + + } + } diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 11054566985..898f014cea3 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -447,8 +447,8 @@ void assertAttributes2(const AttributeGuardList &attributes) { EXPECT_EQUAL(2u, attributes.size()); - EXPECT_EQUAL("attr2", attributes[0]->getName()); - EXPECT_EQUAL("attr1", attributes[1]->getName()); + EXPECT_EQUAL("attr1", attributes[0]->getName()); + EXPECT_EQUAL("attr2", attributes[1]->getName()); } void @@ -833,8 +833,8 @@ requireThatAttributesArePopulatedDuringReprocessing(FixtureType &f) std::vector<AttributeGuard> attrs; f.getAttributeManager()->getAttributeList(attrs); EXPECT_EQUAL(2u, attrs.size()); - TEST_DO(assertAttribute1(attrs[1], CFG_SERIAL, 40)); - TEST_DO(assertAttribute2(attrs[0], 40, 40)); + TEST_DO(assertAttribute1(attrs[0], CFG_SERIAL, 40)); + TEST_DO(assertAttribute2(attrs[1], 40, 40)); } } diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 30fed6fa49e..6c9ffc210a1 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -55,7 +55,6 @@ using storage::spi::RemoveResult; using storage::spi::Result; using storage::spi::Timestamp; using storage::spi::UpdateResult; -using vespalib::BlockingThreadStackExecutor; using vespalib::ThreadStackExecutor; using vespalib::ThreadStackExecutorBase; using vespalib::makeClosure; diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 264cf6d8cfa..8cc075773f7 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -48,7 +48,6 @@ using search::memoryindex::FieldIndexCollection; using search::queryeval::Source; using std::set; using std::string; -using vespalib::BlockingThreadStackExecutor; using vespalib::ThreadStackExecutor; using vespalib::makeLambdaTask; using std::chrono::duration_cast; diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index dfb1268aaa6..c26b008f769 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -243,7 +243,6 @@ struct MyLog struct MyProtonConfigurerOwner : public IProtonConfigurerOwner, public MyLog { - using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; vespalib::ThreadStackExecutor _executor; std::map<DocTypeName, std::shared_ptr<MyDocumentDBConfigOwner>> _dbs; @@ -254,9 +253,9 @@ struct MyProtonConfigurerOwner : public IProtonConfigurerOwner, _dbs() { } - virtual ~MyProtonConfigurerOwner() { } + ~MyProtonConfigurerOwner() { } - virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName, + std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName, document::BucketSpace bucketSpace, const vespalib::string &configId, const std::shared_ptr<BootstrapConfig> &bootstrapConfig, @@ -275,14 +274,14 @@ struct MyProtonConfigurerOwner : public IProtonConfigurerOwner, _log.push_back(os.str()); return db; } - virtual void removeDocumentDB(const DocTypeName &docTypeName) override { + void removeDocumentDB(const DocTypeName &docTypeName) override { ASSERT_FALSE(_dbs.find(docTypeName) == _dbs.end()); _dbs.erase(docTypeName); std::ostringstream os; os << "remove db " << docTypeName.getName(); _log.push_back(os.str()); } - virtual void applyConfig(const std::shared_ptr<BootstrapConfig> &bootstrapConfig) override { + void applyConfig(const std::shared_ptr<BootstrapConfig> &bootstrapConfig) override { std::ostringstream os; os << "apply config " << bootstrapConfig->getGeneration(); _log.push_back(os.str()); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 93d8ef3fff4..3e9ef787f74 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -1,24 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentmetastore.h" -#include "search_context.h" #include "documentmetastoresaver.h" +#include "search_context.h" +#include <vespa/fastos/file.h> +#include <vespa/searchcore/proton/bucketdb/bucketsessionbase.h> +#include <vespa/searchcore/proton/bucketdb/joinbucketssession.h> +#include <vespa/searchcore/proton/bucketdb/splitbucketsession.h> #include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/searchlib/attribute/load_utils.h> #include <vespa/searchlib/attribute/readerbase.h> +#include <vespa/searchlib/common/i_gid_to_lid_mapper.h> +#include <vespa/searchlib/query/query_term_simple.h> #include <vespa/vespalib/btree/btree.hpp> -#include <vespa/vespalib/btree/btreenodestore.hpp> +#include <vespa/vespalib/btree/btreebuilder.hpp> #include <vespa/vespalib/btree/btreenodeallocator.hpp> +#include <vespa/vespalib/btree/btreenodestore.hpp> #include <vespa/vespalib/btree/btreeroot.hpp> -#include <vespa/vespalib/btree/btreebuilder.hpp> -#include <vespa/searchlib/common/i_gid_to_lid_mapper.h> -#include <vespa/searchcore/proton/bucketdb/bucketsessionbase.h> -#include <vespa/searchcore/proton/bucketdb/joinbucketssession.h> -#include <vespa/searchcore/proton/bucketdb/splitbucketsession.h> -#include <vespa/searchlib/query/query_term_simple.h> #include <vespa/vespalib/util/bufferwriter.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/rcuvector.hpp> -#include <vespa/fastos/file.h> #include "document_meta_store_versions.h" #include <vespa/log/log.h> @@ -32,7 +33,7 @@ using search::FileReader; using search::GrowStrategy; using search::IAttributeSaveTarget; using search::LidUsageStats; -using vespalib::MemoryUsage; +using search::attribute::LoadUtils; using search::attribute::SearchContextParams; using search::btree::BTreeNoLeafData; using search::fef::TermFieldMatchData; @@ -42,6 +43,7 @@ using storage::spi::Timestamp; using vespalib::GenerationHandler; using vespalib::GenerationHeldBase; using vespalib::IllegalStateException; +using vespalib::MemoryUsage; using vespalib::make_string; namespace proton { @@ -260,7 +262,7 @@ DocumentMetaStore::readNextDoc(documentmetastore::Reader & reader, TreeType::Bui bool DocumentMetaStore::onLoad() { - documentmetastore::Reader reader(openDAT()); + documentmetastore::Reader reader(LoadUtils::openDAT(*this)); unload(); size_t numElems = reader.getNumElems(); size_t docIdLimit = reader.getDocIdLimit(); diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp index 710c072aa53..d4204473578 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp @@ -7,16 +7,19 @@ namespace proton { void ExecutorMetrics::update(const vespalib::ThreadStackExecutorBase::Stats &stats) { - maxPending.set(stats.maxPendingTasks); + maxPending.set(stats.queueSize.max()); accepted.inc(stats.acceptedTasks); rejected.inc(stats.rejectedTasks); + const auto & qSize = stats.queueSize; + queueSize.addValueBatch(qSize.average(), qSize.count(), qSize.min(), qSize.max()); } ExecutorMetrics::ExecutorMetrics(const std::string &name, metrics::MetricSet *parent) : metrics::MetricSet(name, {}, "Instance specific thread executor metrics", parent), maxPending("maxpending", {}, "Maximum number of pending (active + queued) tasks", this), accepted("accepted", {}, "Number of accepted tasks", this), - rejected("rejected", {}, "Number of rejected tasks", this) + rejected("rejected", {}, "Number of rejected tasks", this), + queueSize("queuesize", {}, "Size of task queue", this) { } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h index a347edffd4b..6b638391d1e 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h @@ -11,9 +11,10 @@ namespace proton { struct ExecutorMetrics : metrics::MetricSet { - metrics::LongValueMetric maxPending; + metrics::LongValueMetric maxPending; // TODO Remove on Vespa 8 or sooner if possible. metrics::LongCountMetric accepted; metrics::LongCountMetric rejected; + metrics::LongAverageMetric queueSize; void update(const vespalib::ThreadStackExecutorBase::Stats &stats); ExecutorMetrics(const std::string &name, metrics::MetricSet *parent); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index f296e264903..1a1d97a657b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -85,7 +85,7 @@ private: DocumentStoreCacheStats() : total(), readySubDb(), notReadySubDb(), removedSubDb() {} }; - using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; + using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>; using IFlushTargetList = std::vector<std::shared_ptr<searchcorespi::IFlushTarget>>; using StatusReportUP = std::unique_ptr<StatusReport>; using ProtonConfig = const vespa::config::search::core::internal::InternalProtonType; diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp index 6ca385711b0..b13fa2baed3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp @@ -2,13 +2,14 @@ #include "executor_thread_service.h" #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/vespalib/util/gate.h> #include <vespa/fastos/thread.h> using vespalib::makeLambdaTask; using vespalib::Executor; using vespalib::Gate; using vespalib::Runnable; -using vespalib::ThreadStackExecutorBase; +using vespalib::SyncableThreadExecutor; namespace proton { @@ -28,7 +29,7 @@ sampleThreadId(FastOS_ThreadId *threadId) } std::unique_ptr<internal::ThreadId> -getThreadId(ThreadStackExecutorBase &executor) +getThreadId(SyncableThreadExecutor &executor) { std::unique_ptr<internal::ThreadId> id = std::make_unique<internal::ThreadId>(); executor.execute(makeLambdaTask([threadId=&id->_id] { sampleThreadId(threadId);})); @@ -45,7 +46,7 @@ runRunnable(Runnable *runnable, Gate *gate) } // namespace -ExecutorThreadService::ExecutorThreadService(ThreadStackExecutorBase &executor) +ExecutorThreadService::ExecutorThreadService(SyncableThreadExecutor &executor) : _executor(executor), _threadId(getThreadId(executor)) { diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h index ccdfb6b72cd..26069b4b8dd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h @@ -2,7 +2,7 @@ #pragma once #include <vespa/searchcorespi/index/i_thread_service.h> -#include <vespa/vespalib/util/threadstackexecutorbase.h> +#include <vespa/vespalib/util/threadexecutor.h> namespace proton { @@ -14,11 +14,11 @@ namespace internal { struct ThreadId; } class ExecutorThreadService : public searchcorespi::index::IThreadService { private: - vespalib::ThreadStackExecutorBase &_executor; + vespalib::SyncableThreadExecutor &_executor; std::unique_ptr<internal::ThreadId> _threadId; public: - ExecutorThreadService(vespalib::ThreadStackExecutorBase &executor); + ExecutorThreadService(vespalib::SyncableThreadExecutor &executor); ~ExecutorThreadService(); Stats getStats() override; @@ -31,6 +31,10 @@ public: _executor.sync(); return *this; } + ExecutorThreadService & shutdown() override { + _executor.shutdown(); + return *this; + } bool isCurrentThread() const override; size_t getNumThreads() const override { return _executor.getNumThreads(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index 6e7b4967f6d..a725b00d485 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -3,23 +3,42 @@ #include "executorthreadingservice.h" #include <vespa/searchcore/proton/metrics/executor_threading_service_stats.h> #include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/vespalib/util/singleexecutor.h> +#include <vespa/vespalib/util/blockingthreadstackexecutor.h> -using vespalib::ThreadStackExecutorBase; + +using vespalib::SyncableThreadExecutor; +using vespalib::BlockingThreadStackExecutor; +using vespalib::SingleExecutor; using search::SequencedTaskExecutor; +using OptimizeFor = vespalib::Executor::OptimizeFor; namespace proton { -ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadStackExecutorBase & sharedExecutor, +namespace { + +std::unique_ptr<SyncableThreadExecutor> +createExecutorWithOneThread(uint32_t stackSize, uint32_t taskLimit, OptimizeFor optimize) { + if (optimize == OptimizeFor::THROUGHPUT) { + return std::make_unique<SingleExecutor>(taskLimit); + } else { + return std::make_unique<BlockingThreadStackExecutor>(1, stackSize, taskLimit); + } +} + +} + +ExecutorThreadingService::ExecutorThreadingService(vespalib::SyncableThreadExecutor & sharedExecutor, uint32_t threads, uint32_t stackSize, uint32_t taskLimit, OptimizeFor optimize) : _sharedExecutor(sharedExecutor), _masterExecutor(1, stackSize), - _indexExecutor(1, stackSize, taskLimit), - _summaryExecutor(1, stackSize, taskLimit), + _indexExecutor(createExecutorWithOneThread(stackSize, taskLimit, optimize)), + _summaryExecutor(createExecutorWithOneThread(stackSize, taskLimit, optimize)), _masterService(_masterExecutor), - _indexService(_indexExecutor), - _summaryService(_summaryExecutor), + _indexService(*_indexExecutor), + _summaryService(*_summaryExecutor), _indexFieldInverter(SequencedTaskExecutor::create(threads, taskLimit)), _indexFieldWriter(SequencedTaskExecutor::create(threads, taskLimit)), _attributeFieldWriter(SequencedTaskExecutor::create(threads, taskLimit, optimize)) @@ -36,8 +55,8 @@ ExecutorThreadingService::sync() _masterExecutor.sync(); } _attributeFieldWriter->sync(); - _indexExecutor.sync(); - _summaryExecutor.sync(); + _indexExecutor->sync(); + _summaryExecutor->sync(); _indexFieldInverter->sync(); _indexFieldWriter->sync(); if (!isMasterThread) { @@ -52,10 +71,10 @@ ExecutorThreadingService::shutdown() _masterExecutor.shutdown(); _masterExecutor.sync(); _attributeFieldWriter->sync(); - _summaryExecutor.shutdown(); - _summaryExecutor.sync(); - _indexExecutor.shutdown(); - _indexExecutor.sync(); + _summaryExecutor->shutdown(); + _summaryExecutor->sync(); + _indexExecutor->shutdown(); + _indexExecutor->sync(); _indexFieldInverter->sync(); _indexFieldWriter->sync(); } @@ -63,8 +82,8 @@ ExecutorThreadingService::shutdown() void ExecutorThreadingService::setTaskLimit(uint32_t taskLimit, uint32_t summaryTaskLimit) { - _indexExecutor.setTaskLimit(taskLimit); - _summaryExecutor.setTaskLimit(summaryTaskLimit); + _indexExecutor->setTaskLimit(taskLimit); + _summaryExecutor->setTaskLimit(summaryTaskLimit); _indexFieldInverter->setTaskLimit(taskLimit); _indexFieldWriter->setTaskLimit(taskLimit); _attributeFieldWriter->setTaskLimit(taskLimit); @@ -74,8 +93,8 @@ ExecutorThreadingServiceStats ExecutorThreadingService::getStats() { return ExecutorThreadingServiceStats(_masterExecutor.getStats(), - _indexExecutor.getStats(), - _summaryExecutor.getStats(), + _indexExecutor->getStats(), + _summaryExecutor->getStats(), _sharedExecutor.getStats(), _indexFieldInverter->getStats(), _indexFieldWriter->getStats(), diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h index 2e4dd2035f3..4d018e2b6f3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h @@ -3,7 +3,6 @@ #include "executor_thread_service.h" #include <vespa/searchcorespi/index/ithreadingservice.h> -#include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/threadstackexecutor.h> namespace proton { @@ -17,16 +16,16 @@ class ExecutorThreadingServiceStats; class ExecutorThreadingService : public searchcorespi::index::IThreadingService { private: - vespalib::ThreadStackExecutorBase & _sharedExecutor; - vespalib::ThreadStackExecutor _masterExecutor; - vespalib::BlockingThreadStackExecutor _indexExecutor; - vespalib::BlockingThreadStackExecutor _summaryExecutor; - ExecutorThreadService _masterService; - ExecutorThreadService _indexService; - ExecutorThreadService _summaryService; - std::unique_ptr<search::ISequencedTaskExecutor> _indexFieldInverter; - std::unique_ptr<search::ISequencedTaskExecutor> _indexFieldWriter; - std::unique_ptr<search::ISequencedTaskExecutor> _attributeFieldWriter; + vespalib::SyncableThreadExecutor & _sharedExecutor; + vespalib::ThreadStackExecutor _masterExecutor; + std::unique_ptr<vespalib::SyncableThreadExecutor> _indexExecutor; + std::unique_ptr<vespalib::SyncableThreadExecutor> _summaryExecutor; + ExecutorThreadService _masterService; + ExecutorThreadService _indexService; + ExecutorThreadService _summaryService; + std::unique_ptr<search::ISequencedTaskExecutor> _indexFieldInverter; + std::unique_ptr<search::ISequencedTaskExecutor> _indexFieldWriter; + std::unique_ptr<search::ISequencedTaskExecutor> _attributeFieldWriter; public: using OptimizeFor = vespalib::Executor::OptimizeFor; @@ -36,7 +35,7 @@ public: * @stackSize The size of the stack of the underlying executors. * @taskLimit The task limit for the index executor. */ - ExecutorThreadingService(vespalib::ThreadStackExecutorBase &sharedExecutor, + ExecutorThreadingService(vespalib::SyncableThreadExecutor &sharedExecutor, uint32_t threads = 1, uint32_t stackSize = 128 * 1024, uint32_t taskLimit = 1000, @@ -56,11 +55,11 @@ public: vespalib::ThreadStackExecutorBase &getMasterExecutor() { return _masterExecutor; } - vespalib::ThreadStackExecutorBase &getIndexExecutor() { - return _indexExecutor; + vespalib::SyncableThreadExecutor &getIndexExecutor() { + return *_indexExecutor; } - vespalib::ThreadStackExecutorBase &getSummaryExecutor() { - return _summaryExecutor; + vespalib::SyncableThreadExecutor &getSummaryExecutor() { + return *_summaryExecutor; } /** @@ -76,7 +75,7 @@ public: searchcorespi::index::IThreadService &summary() override { return _summaryService; } - vespalib::ThreadExecutor &shared() override { + vespalib::SyncableThreadExecutor &shared() override { return _sharedExecutor; } diff --git a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h index fec8430e41d..5a457b168ec 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h @@ -18,8 +18,8 @@ class DocumentDBConfigOwner; */ class IProtonConfigurerOwner { - using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; public: + using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>; virtual ~IProtonConfigurerOwner() { } virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName, document::BucketSpace bucketSpace, diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 20de5bb07c1..3f8db3f2ff9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -32,6 +32,7 @@ #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/util/random.h> #include <vespa/vespalib/net/state_server.h> +#include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/searchlib/aggregation/forcelink.hpp> #include <vespa/searchlib/expression/forcelink.hpp> diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 410f45162e4..4c9d4c77cc4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -57,7 +57,7 @@ private: typedef search::engine::MonitorClient MonitorClient; typedef std::map<DocTypeName, DocumentDB::SP> DocumentDBMap; typedef BootstrapConfig::ProtonConfigSP ProtonConfigSP; - using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; + using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>; using BucketSpace = document::BucketSpace; struct MetricsUpdateHook : metrics::UpdateHook diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp index 0b9293a4aab..45e3c978dd9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp @@ -39,7 +39,7 @@ getBucketSpace(const BootstrapConfig &bootstrapConfig, const DocTypeName &name) } -ProtonConfigurer::ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor, +ProtonConfigurer::ProtonConfigurer(vespalib::SyncableThreadExecutor &executor, IProtonConfigurerOwner &owner, const std::unique_ptr<IProtonDiskLayout> &diskLayout) : IProtonConfigurer(), diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h index c896f12bd4f..54399a26365 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h @@ -25,7 +25,7 @@ class IProtonDiskLayout; class ProtonConfigurer : public IProtonConfigurer { using DocumentDBs = std::map<DocTypeName, std::pair<std::weak_ptr<IDocumentDBConfigOwner>, std::weak_ptr<DocumentDBDirectoryHolder>>>; - using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; + using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>; ExecutorThreadService _executor; IProtonConfigurerOwner &_owner; @@ -48,11 +48,11 @@ class ProtonConfigurer : public IProtonConfigurer void pruneInitialDocumentDBDirs(const ProtonConfigSnapshot &configSnapshot); public: - ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor, + ProtonConfigurer(vespalib::SyncableThreadExecutor &executor, IProtonConfigurerOwner &owner, const std::unique_ptr<IProtonDiskLayout> &diskLayout); - ~ProtonConfigurer(); + ~ProtonConfigurer() override; void setAllowReconfig(bool allowReconfig); diff --git a/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h index 766bdeeefb0..127b696c4ab 100644 --- a/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h @@ -31,6 +31,10 @@ public: _service.sync(); return *this; } + ThreadServiceObserver &shutdown() override { + _service.shutdown(); + return *this; + } bool isCurrentThread() const override { return _service.isCurrentThread(); } diff --git a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h index 7ac9c0c68f2..23c62d179b1 100644 --- a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h @@ -69,6 +69,7 @@ public: search::ISequencedTaskExecutor &attributeFieldWriter() override { return _attributeFieldWriter; } + }; } diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index aaf8f91387e..a76baeced04 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -90,6 +90,7 @@ vespa_define_module( src/tests/attribute/postinglist src/tests/attribute/postinglistattribute src/tests/attribute/reference_attribute + src/tests/attribute/save_target src/tests/attribute/searchable src/tests/attribute/searchcontext src/tests/attribute/sourceselector @@ -214,6 +215,7 @@ vespa_define_module( src/tests/tensor/dense_tensor_store src/tests/tensor/distance_functions src/tests/tensor/hnsw_index + src/tests/tensor/hnsw_saver src/tests/transactionlog src/tests/transactionlogstress src/tests/true diff --git a/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp b/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp index 31af5945337..45d432c29be 100644 --- a/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp @@ -221,7 +221,7 @@ TEST_P(IntegerCompactionTest, compact) test_enum_store_compaction(); } -INSTANTIATE_TEST_CASE_P(IntegerCompactionTestSet, IntegerCompactionTest, ::testing::Values(CollectionType::SINGLE, CollectionType::ARRAY, CollectionType::WSET)); +VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(IntegerCompactionTestSet, IntegerCompactionTest, ::testing::Values(CollectionType::SINGLE, CollectionType::ARRAY, CollectionType::WSET)); using StringCompactionTest = CompactionTest<StringAttribute>; @@ -230,6 +230,6 @@ TEST_P(StringCompactionTest, compact) test_enum_store_compaction(); } -INSTANTIATE_TEST_CASE_P(StringCompactionTestSet, StringCompactionTest, ::testing::Values(CollectionType::SINGLE, CollectionType::ARRAY, CollectionType::WSET)); +VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(StringCompactionTestSet, StringCompactionTest, ::testing::Values(CollectionType::SINGLE, CollectionType::ARRAY, CollectionType::WSET)); GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index bf829f6607a..41313fc7c53 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -108,6 +108,17 @@ public: } IAttributeFileWriter &udatWriter() override { return _udatWriter; } + bool setup_writer(const vespalib::string& file_suffix, + const vespalib::string& desc) override { + (void) file_suffix; + (void) desc; + abort(); + } + IAttributeFileWriter& get_writer(const vespalib::string& file_suffix) override { + (void) file_suffix; + abort(); + } + bool bufEqual(const Buffer &lhs, const Buffer &rhs) const; bool operator==(const MemAttr &rhs) const; diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index 3a885dda233..43e694f0bcd 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -72,7 +72,7 @@ public: #endif using FloatEnumStoreTestTypes = ::testing::Types<FloatEnumStore, DoubleEnumStore>; -TYPED_TEST_CASE(FloatEnumStoreTest, FloatEnumStoreTestTypes); +VESPA_GTEST_TYPED_TEST_SUITE(FloatEnumStoreTest, FloatEnumStoreTestTypes); TYPED_TEST(FloatEnumStoreTest, numbers_can_be_inserted_and_retrieved) { @@ -452,7 +452,7 @@ LoaderTest<StringEnumStore>::load_values(enumstore::EnumeratedLoaderBase& loader #endif using LoaderTestTypes = ::testing::Types<NumericEnumStore, FloatEnumStore, StringEnumStore>; -TYPED_TEST_CASE(LoaderTest, LoaderTestTypes); +VESPA_GTEST_TYPED_TEST_SUITE(LoaderTest, LoaderTestTypes); TYPED_TEST(LoaderTest, store_is_instantiated_with_enumerated_loader) { diff --git a/searchlib/src/tests/attribute/save_target/CMakeLists.txt b/searchlib/src/tests/attribute/save_target/CMakeLists.txt new file mode 100644 index 00000000000..e127f66579e --- /dev/null +++ b/searchlib/src/tests/attribute/save_target/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_attribute_save_target_test_app TEST + SOURCES + attribute_save_target_test.cpp + DEPENDS + searchlib + gtest +) +vespa_add_test(NAME searchlib_attribute_save_target_test_app COMMAND searchlib_attribute_save_target_test_app) diff --git a/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp new file mode 100644 index 00000000000..c746a0aa120 --- /dev/null +++ b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp @@ -0,0 +1,148 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/attribute/attributefilesavetarget.h> +#include <vespa/searchlib/attribute/attributememorysavetarget.h> +#include <vespa/searchlib/common/tunefileinfo.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/test/directory_handler.h> +#include <vespa/searchlib/util/fileutil.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/vespalib/util/exceptions.h> + +#include <vespa/log/log.h> +LOG_SETUP("attribute_save_target_test"); + +using namespace search; +using namespace search::attribute; + +using search::index::DummyFileHeaderContext; +using search::test::DirectoryHandler; + +const vespalib::string test_dir = "test_data/"; + +class SaveTargetTest : public ::testing::Test { +public: + DirectoryHandler dir_handler; + TuneFileAttributes tune_file; + DummyFileHeaderContext file_header_ctx; + IAttributeSaveTarget& target; + vespalib::string base_file_name; + + SaveTargetTest(IAttributeSaveTarget& target_in) + : dir_handler(test_dir), + tune_file(), + file_header_ctx(), + target(target_in), + base_file_name(test_dir + "test_file") + { + } + ~SaveTargetTest() {} + void set_header(const vespalib::string& file_name) { + target.setHeader(AttributeHeader(file_name)); + } + IAttributeFileWriter& setup_writer(const vespalib::string& file_suffix, + const vespalib::string& desc) { + bool res = target.setup_writer(file_suffix, desc); + assert(res); + return target.get_writer(file_suffix); + } + void setup_writer_and_fill(const vespalib::string& file_suffix, + const vespalib::string& desc, + int value) { + auto& writer = setup_writer(file_suffix, desc); + auto buf = writer.allocBufferWriter(); + buf->write(&value, sizeof(int)); + buf->flush(); + } + void validate_loaded_file(const vespalib::string& file_suffix, + const vespalib::string& exp_desc, + int exp_value) + { + vespalib::string file_name = base_file_name + "." + file_suffix; + EXPECT_TRUE(vespalib::fileExists(file_name)); + auto loaded = FileUtil::loadFile(file_name); + EXPECT_FALSE(loaded->empty()); + + const auto& header = loaded->getHeader(); + EXPECT_EQ(file_name, header.getTag("fileName").asString()); + EXPECT_EQ(exp_desc, header.getTag("desc").asString()); + + EXPECT_EQ(sizeof(int), loaded->size()); + int act_value = (reinterpret_cast<const int*>(loaded->buffer()))[0]; + EXPECT_EQ(exp_value, act_value); + } +}; + +class FileSaveTargetTest : public SaveTargetTest { +public: + AttributeFileSaveTarget file_target; + + FileSaveTargetTest() + : SaveTargetTest(file_target), + file_target(tune_file, file_header_ctx) + { + set_header(base_file_name); + } +}; + +TEST_F(FileSaveTargetTest, can_setup_and_return_writers) +{ + setup_writer_and_fill("my1", "desc 1", 123); + setup_writer_and_fill("my2", "desc 2", 456); + target.close(); + + validate_loaded_file("my1", "desc 1", 123); + validate_loaded_file("my2", "desc 2", 456); +} + +TEST_F(FileSaveTargetTest, setup_fails_if_writer_already_exists) +{ + setup_writer("my", "my desc"); + EXPECT_FALSE(target.setup_writer("my", "my desc")); +} + +TEST_F(FileSaveTargetTest, get_throws_if_writer_does_not_exists) +{ + EXPECT_THROW(target.get_writer("na"), vespalib::IllegalArgumentException); +} + +class MemorySaveTargetTest : public SaveTargetTest { +public: + AttributeMemorySaveTarget memory_target; + + MemorySaveTargetTest() + : SaveTargetTest(memory_target), + memory_target() + { + set_header(base_file_name); + } + void write_to_file() { + bool res = memory_target.writeToFile(tune_file, file_header_ctx); + ASSERT_TRUE(res); + } +}; + +TEST_F(MemorySaveTargetTest, can_setup_and_return_writers) +{ + setup_writer_and_fill("my1", "desc 1", 123); + setup_writer_and_fill("my2", "desc 2", 456); + write_to_file(); + + validate_loaded_file("my1", "desc 1", 123); + validate_loaded_file("my2", "desc 2", 456); +} + +TEST_F(MemorySaveTargetTest, setup_fails_if_writer_already_exists) +{ + setup_writer("my", "my desc"); + EXPECT_FALSE(target.setup_writer("my", "my desc")); +} + +TEST_F(MemorySaveTargetTest, get_throws_if_writer_does_not_exists) +{ + EXPECT_THROW(target.get_writer("na"), vespalib::IllegalArgumentException); +} + +GTEST_MAIN_RUN_ALL_TESTS() + diff --git a/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt b/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt index 3794fd88fc3..44ff45d02d3 100644 --- a/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt +++ b/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt @@ -5,5 +5,4 @@ vespa_add_executable(searchlib_tensorattribute_test_app TEST DEPENDS searchlib ) -vespa_add_test(NAME searchlib_tensorattribute_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tensorattribute_test.sh - DEPENDS searchlib_tensorattribute_test_app) +vespa_add_test(NAME searchlib_tensorattribute_test_app COMMAND searchlib_tensorattribute_test_app) diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index d9a4431f89b..39a7e53ca8c 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -5,8 +5,8 @@ #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/tensor/tensor.h> #include <vespa/fastos/file.h> -#include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> +#include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/doc_vector_access.h> @@ -14,11 +14,15 @@ #include <vespa/searchlib/tensor/hnsw_index.h> #include <vespa/searchlib/tensor/nearest_neighbor_index.h> #include <vespa/searchlib/tensor/nearest_neighbor_index_factory.h> +#include <vespa/searchlib/tensor/nearest_neighbor_index_saver.h> #include <vespa/searchlib/tensor/tensor_attribute.h> +#include <vespa/searchlib/test/directory_handler.h> +#include <vespa/searchlib/util/fileutil.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/bufferwriter.h> #include <vespa/log/log.h> LOG_SETUP("tensorattribute_test"); @@ -33,8 +37,10 @@ using search::tensor::DenseTensorAttribute; using search::tensor::DocVectorAccess; using search::tensor::GenericTensorAttribute; using search::tensor::HnswIndex; +using search::tensor::HnswNode; using search::tensor::NearestNeighborIndex; using search::tensor::NearestNeighborIndexFactory; +using search::tensor::NearestNeighborIndexSaver; using search::tensor::TensorAttribute; using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; @@ -75,6 +81,18 @@ vec_2d(double x0, double x1) return TensorSpec(vec_2d_spec).add({{"x", 0}}, x0).add({{"x", 1}}, x1); } +class MockIndexSaver : public NearestNeighborIndexSaver { +private: + int _index_value; + +public: + MockIndexSaver(int index_value) : _index_value(index_value) {} + void save(search::BufferWriter& writer) const override { + writer.write(&_index_value, sizeof(int)); + writer.flush(); + } +}; + class MockNearestNeighborIndex : public NearestNeighborIndex { private: using Entry = std::pair<uint32_t, DoubleVector>; @@ -86,6 +104,7 @@ private: generation_t _transfer_gen; generation_t _trim_gen; mutable size_t _memory_usage_cnt; + int _index_value; public: MockNearestNeighborIndex(const DocVectorAccess& vectors) @@ -94,13 +113,20 @@ public: _removes(), _transfer_gen(std::numeric_limits<generation_t>::max()), _trim_gen(std::numeric_limits<generation_t>::max()), - _memory_usage_cnt(0) + _memory_usage_cnt(0), + _index_value(0) { } void clear() { _adds.clear(); _removes.clear(); } + int get_index_value() const { + return _index_value; + } + void save_index_with_value(int value) { + _index_value = value; + } void expect_empty_add() const { EXPECT_TRUE(_adds.empty()); } @@ -143,6 +169,17 @@ public: return vespalib::MemoryUsage(); } void get_state(const vespalib::slime::Inserter&) const override {} + std::unique_ptr<NearestNeighborIndexSaver> make_saver() const override { + if (_index_value != 0) { + return std::make_unique<MockIndexSaver>(_index_value); + } + return std::unique_ptr<NearestNeighborIndexSaver>(); + } + bool load(const search::fileutil::LoadedBuffer& buf) override { + ASSERT_EQUAL(sizeof(int), buf.size()); + _index_value = (reinterpret_cast<const int*>(buf.buffer()))[0]; + return true; + } std::vector<Neighbor> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) const override { (void) k; (void) vector; @@ -166,12 +203,15 @@ class MockNearestNeighborIndexFactory : public NearestNeighborIndexFactory { } }; -struct Fixture -{ +const vespalib::string test_dir = "test_data/"; +const vespalib::string attr_name = test_dir + "my_attr"; + +struct Fixture { using BasicType = search::attribute::BasicType; using CollectionType = search::attribute::CollectionType; using Config = search::attribute::Config; + search::test::DirectoryHandler _dir_handler; Config _cfg; vespalib::string _name; vespalib::string _typeSpec; @@ -185,8 +225,9 @@ struct Fixture bool useDenseTensorAttribute = false, bool enable_hnsw_index = false, bool use_mock_index = false) - : _cfg(BasicType::TENSOR, CollectionType::SINGLE), - _name("test"), + : _dir_handler(test_dir), + _cfg(BasicType::TENSOR, CollectionType::SINGLE), + _name(attr_name), _typeSpec(typeSpec), _index_factory(std::make_unique<DefaultNearestNeighborIndexFactory>()), _tensorAttr(), @@ -225,11 +266,20 @@ struct Fixture return *result; } - MockNearestNeighborIndex& mock_index() { + template <typename IndexType> + IndexType& get_nearest_neighbor_index() { assert(as_dense_tensor().nearest_neighbor_index() != nullptr); - auto mock_index = dynamic_cast<const MockNearestNeighborIndex*>(as_dense_tensor().nearest_neighbor_index()); - assert(mock_index != nullptr); - return *const_cast<MockNearestNeighborIndex*>(mock_index); + auto index = dynamic_cast<const IndexType*>(as_dense_tensor().nearest_neighbor_index()); + assert(index != nullptr); + return *const_cast<IndexType*>(index); + } + + HnswIndex& hnsw_index() { + return get_nearest_neighbor_index<HnswIndex>(); + } + + MockNearestNeighborIndex& mock_index() { + return get_nearest_neighbor_index<MockNearestNeighborIndex>(); } void ensureSpace(uint32_t docId) { @@ -322,7 +372,6 @@ struct Fixture void testEmptyTensor(); }; - void Fixture::testEmptyAttribute() { @@ -383,7 +432,6 @@ Fixture::testSaveLoad() TEST_DO(assertGetNoTensor(2)); } - void Fixture::testCompaction() { @@ -438,7 +486,8 @@ Fixture::testTensorTypeFileHeaderTag() vespalib::FileHeader header; FastOS_File file; - EXPECT_TRUE(file.OpenReadOnly("test.dat")); + vespalib::string file_name = attr_name + ".dat"; + EXPECT_TRUE(file.OpenReadOnly(file_name.c_str())); (void) header.readFile(file); file.Close(); EXPECT_TRUE(header.hasTag("tensortype")); @@ -450,7 +499,6 @@ Fixture::testTensorTypeFileHeaderTag() } } - void Fixture::testEmptyTensor() { @@ -465,7 +513,6 @@ Fixture::testEmptyTensor() } } - template <class MakeFixture> void testAll(MakeFixture &&f) { @@ -499,21 +546,49 @@ TEST_F("Hnsw index is NOT instantiated in dense tensor attribute by default", EXPECT_TRUE(tensor.nearest_neighbor_index() == nullptr); } -TEST_F("Hnsw index is instantiated in dense tensor attribute when specified in config", - Fixture(vec_2d_spec, true, true)) +class DenseTensorAttributeHnswIndex : public Fixture { +public: + DenseTensorAttributeHnswIndex() : Fixture(vec_2d_spec, true, true, false) {} +}; + +TEST_F("Hnsw index is instantiated in dense tensor attribute when specified in config", DenseTensorAttributeHnswIndex) { - const auto& tensor = f.as_dense_tensor(); - ASSERT_TRUE(tensor.nearest_neighbor_index() != nullptr); - auto hnsw_index = dynamic_cast<const HnswIndex*>(tensor.nearest_neighbor_index()); - ASSERT_TRUE(hnsw_index != nullptr); + auto& index = f.hnsw_index(); - const auto& cfg = hnsw_index->config(); + const auto& cfg = index.config(); EXPECT_EQUAL(8u, cfg.max_links_at_level_0()); EXPECT_EQUAL(4u, cfg.max_links_on_inserts()); EXPECT_EQUAL(20u, cfg.neighbors_to_explore_at_construction()); EXPECT_TRUE(cfg.heuristic_select_neighbors()); } +void +expect_level_0(uint32_t exp_docid, const HnswNode& node) +{ + ASSERT_GREATER_EQUAL(node.size(), 1u); + ASSERT_EQUAL(1u, node.level(0).size()); + EXPECT_EQUAL(exp_docid, node.level(0)[0]); +} + +TEST_F("Hnsw index is integrated in dense tensor attribute and can be saved and loaded", DenseTensorAttributeHnswIndex) +{ + // Set two points that will be linked together in level 0 of the hnsw graph. + f.set_tensor(1, vec_2d(3, 5)); + f.set_tensor(2, vec_2d(7, 9)); + + auto &index_a = f.hnsw_index(); + expect_level_0(2, index_a.get_node(1)); + expect_level_0(1, index_a.get_node(2)); + f.save(); + EXPECT_TRUE(vespalib::fileExists(attr_name + ".nnidx")); + + f.load(); + auto &index_b = f.hnsw_index(); + EXPECT_NOT_EQUAL(&index_a, &index_b); + expect_level_0(2, index_b.get_node(1)); + expect_level_0(1, index_b.get_node(2)); +} + class DenseTensorAttributeMockIndex : public Fixture { public: DenseTensorAttributeMockIndex() : Fixture(vec_2d_spec, true, true, true) {} @@ -551,17 +626,6 @@ TEST_F("clearDoc() updates nearest neighbor index", DenseTensorAttributeMockInde index.expect_empty_add(); } -TEST_F("onLoad() updates nearest neighbor index", DenseTensorAttributeMockIndex) -{ - f.set_tensor(1, vec_2d(3, 5)); - f.set_tensor(2, vec_2d(7, 9)); - f.save(); - f.load(); - auto& index = f.mock_index(); - index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); -} - - TEST_F("commit() ensures transfer and trim hold lists on nearest neighbor index", DenseTensorAttributeMockIndex) { auto& index = f.mock_index(); @@ -598,4 +662,32 @@ TEST_F("Memory usage is extracted from index when updating stats on attribute", EXPECT_EQUAL(before + 1, after); } -TEST_MAIN() { TEST_RUN_ALL(); vespalib::unlink("test.dat"); } +TEST_F("Nearest neighbor index can be saved to disk and then loaded from file", DenseTensorAttributeMockIndex) +{ + f.set_tensor(1, vec_2d(3, 5)); + f.set_tensor(2, vec_2d(7, 9)); + f.mock_index().save_index_with_value(123); + f.save(); + EXPECT_TRUE(vespalib::fileExists(attr_name + ".nnidx")); + + f.load(); // index is loaded from saved file + auto& index = f.mock_index(); + EXPECT_EQUAL(123, index.get_index_value()); + index.expect_adds({}); +} + +TEST_F("onLoad() reconstructs nearest neighbor index if save file does not exists", DenseTensorAttributeMockIndex) +{ + f.set_tensor(1, vec_2d(3, 5)); + f.set_tensor(2, vec_2d(7, 9)); + f.save(); + EXPECT_FALSE(vespalib::fileExists(attr_name + ".nnidx")); + + f.load(); // index is reconstructed by adding all loaded tensors + auto& index = f.mock_index(); + EXPECT_EQUAL(0, index.get_index_value()); + index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); +} + +TEST_MAIN() { TEST_RUN_ALL(); } + diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.sh b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.sh deleted file mode 100755 index dd9399dea78..00000000000 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -set -e -$VALGRIND ./searchlib_tensorattribute_test_app -rm -rf *.dat diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt index 6c593d20683..fd6cd9efc43 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt +++ b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt @@ -13,3 +13,11 @@ vespa_add_executable(searchlib_sequencedtaskexecutor_test_app TEST searchlib ) vespa_add_test(NAME searchlib_sequencedtaskexecutor_test_app COMMAND searchlib_sequencedtaskexecutor_test_app) + +vespa_add_executable(searchlib_adaptive_sequenced_executor_test_app TEST + SOURCES + adaptive_sequenced_executor_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_adaptive_sequenced_executor_test_app COMMAND searchlib_adaptive_sequenced_executor_test_app) diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp new file mode 100644 index 00000000000..ba66b28108c --- /dev/null +++ b/searchlib/src/tests/common/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp @@ -0,0 +1,251 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/common/adaptive_sequenced_executor.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/test/insertion_operators.h> + +#include <mutex> +#include <condition_variable> +#include <unistd.h> + +#include <vespa/log/log.h> +LOG_SETUP("adaptive_sequenced_executor_test"); + +namespace search::common { + + +class Fixture +{ +public: + AdaptiveSequencedExecutor _threads; + + Fixture() : _threads(2, 2, 0, 1000) { } +}; + + +class TestObj +{ +public: + std::mutex _m; + std::condition_variable _cv; + int _done; + int _fail; + int _val; + + TestObj() + : _m(), + _cv(), + _done(0), + _fail(0), + _val(0) + { + } + + void + modify(int oldValue, int newValue) + { + { + std::lock_guard<std::mutex> guard(_m); + if (_val == oldValue) { + _val = newValue; + } else { + ++_fail; + } + ++_done; + } + _cv.notify_all(); + } + + void + wait(int wantDone) + { + std::unique_lock<std::mutex> guard(_m); + _cv.wait(guard, [&] { return this->_done >= wantDone; }); + } +}; + +TEST_F("testExecute", Fixture) { + std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); + EXPECT_EQUAL(0, tv->_val); + f._threads.execute(1, [&]() { tv->modify(0, 42); }); + tv->wait(1); + EXPECT_EQUAL(0, tv->_fail); + EXPECT_EQUAL(42, tv->_val); + f._threads.sync(); + EXPECT_EQUAL(0, tv->_fail); + EXPECT_EQUAL(42, tv->_val); +} + + +TEST_F("require that task with same component id are serialized", Fixture) +{ + std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); + EXPECT_EQUAL(0, tv->_val); + f._threads.execute(0, [&]() { usleep(2000); tv->modify(0, 14); }); + f._threads.execute(0, [&]() { tv->modify(14, 42); }); + tv->wait(2); + EXPECT_EQUAL(0, tv->_fail); + EXPECT_EQUAL(42, tv->_val); + f._threads.sync(); + EXPECT_EQUAL(0, tv->_fail); + EXPECT_EQUAL(42, tv->_val); +} + +TEST_F("require that task with different component ids are not serialized", Fixture) +{ + int tryCnt = 0; + for (tryCnt = 0; tryCnt < 100; ++tryCnt) { + std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); + EXPECT_EQUAL(0, tv->_val); + f._threads.execute(0, [&]() { usleep(2000); tv->modify(0, 14); }); + f._threads.execute(2, [&]() { tv->modify(14, 42); }); + tv->wait(2); + if (tv->_fail != 1) { + continue; + } + EXPECT_EQUAL(1, tv->_fail); + EXPECT_EQUAL(14, tv->_val); + f._threads.sync(); + EXPECT_EQUAL(1, tv->_fail); + EXPECT_EQUAL(14, tv->_val); + break; + } + EXPECT_TRUE(tryCnt < 100); +} + + +TEST_F("require that task with same string component id are serialized", Fixture) +{ + std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); + EXPECT_EQUAL(0, tv->_val); + auto test2 = [&]() { tv->modify(14, 42); }; + f._threads.execute(f._threads.getExecutorId("0"), [&]() { usleep(2000); tv->modify(0, 14); }); + f._threads.execute(f._threads.getExecutorId("0"), test2); + tv->wait(2); + EXPECT_EQUAL(0, tv->_fail); + EXPECT_EQUAL(42, tv->_val); + f._threads.sync(); + EXPECT_EQUAL(0, tv->_fail); + EXPECT_EQUAL(42, tv->_val); +} + +namespace { + +int detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int tryLimit) +{ + int tryCnt = 0; + for (tryCnt = 0; tryCnt < tryLimit; ++tryCnt) { + std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); + EXPECT_EQUAL(0, tv->_val); + f._threads.execute(f._threads.getExecutorId("0"), [&]() { usleep(2000); tv->modify(0, 14); }); + f._threads.execute(f._threads.getExecutorId(altComponentId), [&]() { tv->modify(14, 42); }); + tv->wait(2); + if (tv->_fail != 1) { + continue; + } + EXPECT_EQUAL(1, tv->_fail); + EXPECT_EQUAL(14, tv->_val); + f._threads.sync(); + EXPECT_EQUAL(1, tv->_fail); + EXPECT_EQUAL(14, tv->_val); + break; + } + return tryCnt; +} + +vespalib::string makeAltComponentId(Fixture &f) +{ + int tryCnt = 0; + char altComponentId[20]; + ISequencedTaskExecutor::ExecutorId executorId0 = f._threads.getExecutorId("0"); + for (tryCnt = 1; tryCnt < 100; ++tryCnt) { + sprintf(altComponentId, "%d", tryCnt); + if (f._threads.getExecutorId(altComponentId) == executorId0) { + break; + } + } + EXPECT_TRUE(tryCnt < 100); + return altComponentId; +} + +} + +TEST_F("require that task with different string component ids are not serialized", Fixture) +{ + int tryCnt = detectSerializeFailure(f, "2", 100); + EXPECT_TRUE(tryCnt < 100); +} + + +TEST_F("require that task with different string component ids mapping to the same executor id are serialized", + Fixture) +{ + vespalib::string altComponentId = makeAltComponentId(f); + LOG(info, "second string component id is \"%s\"", altComponentId.c_str()); + int tryCnt = detectSerializeFailure(f, altComponentId, 100); + EXPECT_TRUE(tryCnt == 100); +} + + +TEST_F("require that execute works with const lambda", Fixture) +{ + int i = 5; + std::vector<int> res; + const auto lambda = [i, &res]() mutable + { res.push_back(i--); res.push_back(i--); }; + f._threads.execute(0, lambda); + f._threads.execute(0, lambda); + f._threads.sync(); + std::vector<int> exp({5, 4, 5, 4}); + EXPECT_EQUAL(exp, res); + EXPECT_EQUAL(5, i); +} + +TEST_F("require that execute works with reference to lambda", Fixture) +{ + int i = 5; + std::vector<int> res; + auto lambda = [i, &res]() mutable + { res.push_back(i--); res.push_back(i--); }; + auto &lambdaref = lambda; + f._threads.execute(0, lambdaref); + f._threads.execute(0, lambdaref); + f._threads.sync(); + std::vector<int> exp({5, 4, 5, 4}); + EXPECT_EQUAL(exp, res); + EXPECT_EQUAL(5, i); +} + +TEST_F("require that executeLambda works", Fixture) +{ + int i = 5; + std::vector<int> res; + const auto lambda = [i, &res]() mutable + { res.push_back(i--); res.push_back(i--); }; + f._threads.executeLambda(ISequencedTaskExecutor::ExecutorId(0), lambda); + f._threads.sync(); + std::vector<int> exp({5, 4}); + EXPECT_EQUAL(exp, res); + EXPECT_EQUAL(5, i); +} + +TEST("require that you get correct number of executors") { + AdaptiveSequencedExecutor seven(7, 1, 0, 10); + EXPECT_EQUAL(7u, seven.getNumExecutors()); +} + +TEST("require that you distribute well") { + AdaptiveSequencedExecutor seven(7, 1, 0, 10); + EXPECT_EQUAL(7u, seven.getNumExecutors()); + EXPECT_EQUAL(97u, seven.getComponentHashSize()); + EXPECT_EQUAL(0u, seven.getComponentEffectiveHashSize()); + for (uint32_t id=0; id < 1000; id++) { + EXPECT_EQUAL((id%97)%7, seven.getExecutorId(id).getId()); + } + EXPECT_EQUAL(97u, seven.getComponentHashSize()); + EXPECT_EQUAL(97u, seven.getComponentEffectiveHashSize()); +} + +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp index 9491617c135..362dc28d36a 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp +++ b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp @@ -1,30 +1,70 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/searchlib/common/adaptive_sequenced_executor.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/vespalib/util/time.h> #include <atomic> +using search::ISequencedTaskExecutor; using search::SequencedTaskExecutor; +using search::AdaptiveSequencedExecutor; using ExecutorId = search::ISequencedTaskExecutor::ExecutorId; -int main(int argc, char *argv[]) { - unsigned long numTasks = 1000000; - unsigned numThreads = 4; - unsigned taskLimit = 1000; - vespalib::Executor::OptimizeFor optimize = vespalib::Executor::OptimizeFor::LATENCY; - std::atomic<long> counter(0); - if (argc > 1) - numTasks = atol(argv[1]); - if (argc > 2) - numThreads = atoi(argv[2]); - if (argc > 3) - taskLimit = atoi(argv[3]); - if (argc > 4) - optimize = vespalib::Executor::OptimizeFor::THROUGHPUT; +size_t do_work(size_t size) { + size_t ret = 0; + for (size_t i = 0; i < size; ++i) { + for (size_t j = 0; j < 128; ++j) { + ret = (ret + i) * j; + } + } + return ret; +} - auto executor = SequencedTaskExecutor::create(numThreads, taskLimit, optimize); - for (unsigned long tid(0); tid < numTasks; tid++) { - executor->executeTask(ExecutorId(tid%numThreads), vespalib::makeLambdaTask([&counter] { counter++; })); +struct SimpleParams { + int argc; + char **argv; + int idx; + SimpleParams(int argc_in, char **argv_in) : argc(argc_in), argv(argv_in), idx(0) {} + int next(const char *name, int fallback) { + ++idx; + int value = 0; + if (argc > idx) { + value = atoi(argv[idx]); + } else { + value = fallback; + } + fprintf(stderr, "param %s: %d\n", name, value); + return value; + } +}; + +int main(int argc, char **argv) { + SimpleParams params(argc, argv); + bool use_adaptive_executor = params.next("use_adaptive_executor", 0); + bool optimize_for_throughput = params.next("optimize_for_throughput", 0); + size_t num_tasks = params.next("num_tasks", 1000000); + size_t num_strands = params.next("num_strands", 4); + size_t task_limit = params.next("task_limit", 1000); + size_t num_threads = params.next("num_threads", num_strands); + size_t max_waiting = params.next("max_waiting", optimize_for_throughput ? 32 : 0); + size_t work_size = params.next("work_size", 0); + std::atomic<long> counter(0); + std::unique_ptr<ISequencedTaskExecutor> executor; + if (use_adaptive_executor) { + executor = std::make_unique<AdaptiveSequencedExecutor>(num_strands, num_threads, max_waiting, task_limit); + } else { + auto optimize = optimize_for_throughput + ? vespalib::Executor::OptimizeFor::THROUGHPUT + : vespalib::Executor::OptimizeFor::LATENCY; + executor = SequencedTaskExecutor::create(num_strands, task_limit, optimize); + } + vespalib::Timer timer; + for (size_t task_id = 0; task_id < num_tasks; ++task_id) { + executor->executeTask(ExecutorId(task_id % num_strands), + vespalib::makeLambdaTask([&counter,work_size] { (void) do_work(work_size); counter++; })); } - return 0; + executor.reset(); + fprintf(stderr, "\ntotal time: %zu ms\n", vespalib::count_ms(timer.elapsed())); + return (size_t(counter) == num_tasks) ? 0 : 1; } diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 64a999906d8..1c07c81bc2f 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -1238,10 +1238,11 @@ Test::testDotProduct() assertDotProduct(0, "(f:5,g:5)", 1, "wsextstr"); assertDotProduct(550, "(a:1,b:2,c:3,d:4,e:5)", 1, "wsextstr"); } - for (const char * name : {"wsbyte", "wsint"}) { - assertDotProduct(0, "()", 1, name); - assertDotProduct(0, "(6:5,7:5)", 1, name); - assertDotProduct(55, "(1:1,2:2,3:3,4:4,5:5)", 1, name); + for (const char * name : {"wsbyte", "wsint", "wsint_fast"}) { + TEST_DO(assertDotProduct(0, "()", 1, name)); + TEST_DO(assertDotProduct(0, "(6:5,7:5)", 1, name)); + TEST_DO(assertDotProduct(18, "(4:4.5)", 1, name)); + TEST_DO(assertDotProduct(57, "(1:1,2:2,3:3,4:4.5,5:5)", 1, name)); } for (const char * name : {"arrbyte", "arrint", "arrfloat", "arrint_fast", "arrfloat_fast"}) { assertDotProduct(0, "()", 1, name); @@ -1300,6 +1301,7 @@ Test::setupForDotProductTest(FtFeatureTest & ft) }; std::vector<Config> cfgList = { {"wsint", AVBT::INT32, AVCT::WSET, false}, {"wsbyte", AVBT::INT8, AVCT::WSET, false}, + {"wsint_fast", AVBT::INT8, AVCT::WSET, true}, {"arrbyte", AVBT::INT8, AVCT::ARRAY, false}, {"arrint", AVBT::INT32, AVCT::ARRAY, false}, {"arrfloat", AVBT::FLOAT, AVCT::ARRAY, false}, diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index c562c0cf29c..7a0c240dea5 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -517,7 +517,7 @@ struct FieldIndexTest : public ::testing::Test { }; using FieldIndexTestTypes = ::testing::Types<FieldIndex<false>, FieldIndex<true>>; -TYPED_TEST_CASE(FieldIndexTest, FieldIndexTestTypes); +VESPA_GTEST_TYPED_TEST_SUITE(FieldIndexTest, FieldIndexTestTypes); // Disable warnings emitted by gtest generated files when using typed tests #pragma GCC diagnostic push diff --git a/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt new file mode 100644 index 00000000000..90202e222a7 --- /dev/null +++ b/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_hnsw_save_load_test_app TEST + SOURCES + hnsw_save_load_test.cpp + DEPENDS + searchlib + gtest +) +vespa_add_test(NAME searchlib_hnsw_save_load_test_app COMMAND searchlib_hnsw_save_load_test_app) diff --git a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp new file mode 100644 index 00000000000..b9e27d413f3 --- /dev/null +++ b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp @@ -0,0 +1,150 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/tensor/hnsw_graph.h> +#include <vespa/searchlib/tensor/hnsw_index_saver.h> +#include <vespa/searchlib/tensor/hnsw_index_loader.h> +#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/fileutil.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vector> + +#include <vespa/log/log.h> +LOG_SETUP("hnsw_save_load_test"); + +using namespace search::tensor; +using search::BufferWriter; +using search::fileutil::LoadedBuffer; + +class VectorBufferWriter : public BufferWriter { +private: + char tmp[1024]; +public: + std::vector<char> output; + VectorBufferWriter() { + setup(tmp, 1024); + } + ~VectorBufferWriter() {} + void flush() override { + for (size_t i = 0; i < usedLen(); ++i) { + output.push_back(tmp[i]); + } + rewind(); + } +}; + +using V = std::vector<uint32_t>; + +void populate(HnswGraph &graph) { + // no 0 + graph.make_node_for_document(1, 1); + graph.make_node_for_document(2, 2); + // no 3 + graph.make_node_for_document(4, 2); + graph.make_node_for_document(5, 0); + graph.make_node_for_document(6, 1); + + graph.set_link_array(1, 0, V{2, 4, 6}); + graph.set_link_array(2, 0, V{1, 4, 6}); + graph.set_link_array(4, 0, V{1, 2, 6}); + graph.set_link_array(6, 0, V{1, 2, 4}); + graph.set_link_array(2, 1, V{4}); + graph.set_link_array(4, 1, V{2}); + graph.set_entry_node(2, 1); +} + +void modify(HnswGraph &graph) { + graph.remove_node_for_document(2); + graph.remove_node_for_document(6); + graph.make_node_for_document(7, 2); + + graph.set_link_array(1, 0, V{7, 4}); + graph.set_link_array(4, 0, V{7, 2}); + graph.set_link_array(7, 0, V{4, 2}); + graph.set_link_array(4, 1, V{7}); + graph.set_link_array(7, 1, V{4}); + + graph.set_entry_node(4, 1); +} + + +class CopyGraphTest : public ::testing::Test { +public: + HnswGraph original; + HnswGraph copy; + + void expect_empty_d(uint32_t docid) const { + EXPECT_FALSE(copy.node_refs[docid].load_acquire().valid()); + } + + void expect_level_0(uint32_t docid, const V& exp_links) const { + auto levels = copy.get_level_array(docid); + EXPECT_GE(levels.size(), 1); + auto links = copy.get_link_array(docid, 0); + EXPECT_EQ(exp_links.size(), links.size()); + for (size_t i = 0; i < exp_links.size() && i < links.size(); ++i) { + EXPECT_EQ(exp_links[i], links[i]); + } + } + + void expect_level_1(uint32_t docid, const V& exp_links) const { + auto levels = copy.get_level_array(docid); + EXPECT_EQ(2, levels.size()); + auto links = copy.get_link_array(docid, 1); + EXPECT_EQ(exp_links.size(), links.size()); + for (size_t i = 0; i < exp_links.size() && i < links.size(); ++i) { + EXPECT_EQ(exp_links[i], links[i]); + } + } + + std::vector<char> save_original() const { + HnswIndexSaver saver(original); + VectorBufferWriter vector_writer; + saver.save(vector_writer); + return vector_writer.output; + } + void load_copy(std::vector<char> data) { + HnswIndexLoader loader(copy); + LoadedBuffer buffer(&data[0], data.size()); + loader.load(buffer); + } + + void expect_copy_as_populated() const { + EXPECT_EQ(copy.size(), 7); + EXPECT_EQ(copy.entry_docid, 2); + EXPECT_EQ(copy.entry_level, 1); + + expect_empty_d(0); + expect_empty_d(3); + expect_empty_d(5); + + expect_level_0(1, {2, 4, 6}); + expect_level_0(2, {1, 4, 6}); + expect_level_0(4, {1, 2, 6}); + expect_level_0(6, {1, 2, 4}); + + expect_level_1(2, {4}); + expect_level_1(4, {2}); + } +}; + +TEST_F(CopyGraphTest, reconstructs_graph) +{ + populate(original); + auto data = save_original(); + load_copy(data); + expect_copy_as_populated(); +} + +TEST_F(CopyGraphTest, later_changes_ignored) +{ + populate(original); + HnswIndexSaver saver(original); + modify(original); + VectorBufferWriter vector_writer; + saver.save(vector_writer); + auto data = vector_writer.output; + load_copy(data); + expect_copy_as_populated(); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp index 224d5758028..3d7010ba6c3 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp @@ -22,7 +22,12 @@ const vespalib::string predicateUpperBoundTag = "predicate.upper_bound"; } AttributeHeader::AttributeHeader() - : _fileName(""), + : AttributeHeader("") +{ +} + +AttributeHeader::AttributeHeader(const vespalib::string &fileName) + : _fileName(fileName), _basicType(attribute::BasicType::Type::NONE), _collectionType(attribute::CollectionType::Type::SINGLE), _tensorType(vespalib::eval::ValueType::error_type()), diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.h b/searchlib/src/vespa/searchlib/attribute/attribute_header.h index 303c469e755..24eac8336b4 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_header.h +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.h @@ -35,6 +35,7 @@ private: void internalExtractTags(const vespalib::GenericHeader &header); public: AttributeHeader(); + AttributeHeader(const vespalib::string &fileName); AttributeHeader(const vespalib::string &fileName, BasicType basicType, CollectionType collectionType, diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp index f57094ae592..f284fecbf98 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp @@ -3,14 +3,16 @@ #include "attributefilesavetarget.h" #include "attributevector.h" #include <vespa/searchlib/common/fileheadercontext.h> -#include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/data/databuffer.h> +#include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/util/error.h> +#include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.attributefilesavetarget"); using vespalib::getLastErrorString; +using vespalib::IllegalArgumentException; namespace search { @@ -18,13 +20,16 @@ using common::FileHeaderContext; AttributeFileSaveTarget:: -AttributeFileSaveTarget(const TuneFileAttributes &tuneFileAttributes, - const FileHeaderContext &fileHeaderContext) +AttributeFileSaveTarget(const TuneFileAttributes& tune_file, + const FileHeaderContext& file_header_ctx) : IAttributeSaveTarget(), - _datWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector data file"), - _idxWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector idx file"), - _weightWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector weight file"), - _udatWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector unique data file") + _tune_file(tune_file), + _file_header_ctx(file_header_ctx), + _datWriter(tune_file, file_header_ctx, _header, "Attribute vector data file"), + _idxWriter(tune_file, file_header_ctx, _header, "Attribute vector idx file"), + _weightWriter(tune_file, file_header_ctx, _header, "Attribute vector weight file"), + _udatWriter(tune_file, file_header_ctx, _header, "Attribute vector unique data file"), + _writers() { } @@ -66,23 +71,23 @@ AttributeFileSaveTarget::close() _udatWriter.close(); _idxWriter.close(); _weightWriter.close(); + for (auto& writer : _writers) { + writer.second->close(); + } } - IAttributeFileWriter & AttributeFileSaveTarget::datWriter() { return _datWriter; } - IAttributeFileWriter & AttributeFileSaveTarget::idxWriter() { return _idxWriter; } - IAttributeFileWriter & AttributeFileSaveTarget::weightWriter() { @@ -95,6 +100,33 @@ AttributeFileSaveTarget::udatWriter() return _udatWriter; } +bool +AttributeFileSaveTarget::setup_writer(const vespalib::string& file_suffix, + const vespalib::string& desc) +{ + vespalib::string file_name(_header.getFileName() + "." + file_suffix); + auto writer = std::make_unique<AttributeFileWriter>(_tune_file, _file_header_ctx, + _header, desc); + if (!writer->open(file_name)) { + return false; + } + auto itr = _writers.find(file_suffix); + if (itr != _writers.end()) { + return false; + } + _writers.insert(std::make_pair(file_suffix, std::move(writer))); + return true; +} + +IAttributeFileWriter& +AttributeFileSaveTarget::get_writer(const vespalib::string& file_suffix) +{ + auto itr = _writers.find(file_suffix); + if (itr == _writers.end()) { + throw IllegalArgumentException("File writer with suffix '" + file_suffix + "' does not exist"); + } + return *itr->second; +} } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.h b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.h index acb3daf82e0..9a9d38615ea 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.h +++ b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.h @@ -4,24 +4,30 @@ #include "iattributesavetarget.h" #include "attributefilewriter.h" +#include <vespa/vespalib/stllike/hash_fun.h> +#include <unordered_map> -namespace search -{ +namespace search { /** * Class used to save an attribute vector to file(s). **/ -class AttributeFileSaveTarget : public IAttributeSaveTarget -{ +class AttributeFileSaveTarget : public IAttributeSaveTarget { private: + using FileWriterUP = std::unique_ptr<AttributeFileWriter>; + using WriterMap = std::unordered_map<vespalib::string, FileWriterUP, vespalib::hash<vespalib::string>>; + + const TuneFileAttributes& _tune_file; + const search::common::FileHeaderContext& _file_header_ctx; AttributeFileWriter _datWriter; AttributeFileWriter _idxWriter; AttributeFileWriter _weightWriter; AttributeFileWriter _udatWriter; + WriterMap _writers; public: - AttributeFileSaveTarget(const TuneFileAttributes &tuneFileAttributes, - const search::common::FileHeaderContext &fileHeaderContext); + AttributeFileSaveTarget(const TuneFileAttributes& tune_file, + const search::common::FileHeaderContext& file_header_ctx); ~AttributeFileSaveTarget() override; // Implements IAttributeSaveTarget @@ -35,6 +41,11 @@ public: IAttributeFileWriter &idxWriter() override; IAttributeFileWriter &weightWriter() override; IAttributeFileWriter &udatWriter() override; + + bool setup_writer(const vespalib::string& file_suffix, + const vespalib::string& desc) override; + IAttributeFileWriter& get_writer(const vespalib::string& file_suffix) override; + }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp index 372168143ab..b28887691e5 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp @@ -1,24 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "attributememorysavetarget.h" #include "attributefilesavetarget.h" +#include "attributememorysavetarget.h" #include "attributevector.h" +#include <vespa/vespalib/util/exceptions.h> namespace search { using search::common::FileHeaderContext; +using vespalib::IllegalArgumentException; AttributeMemorySaveTarget::AttributeMemorySaveTarget() : _datWriter(), _idxWriter(), _weightWriter(), - _udatWriter() + _udatWriter(), + _writers() { } -AttributeMemorySaveTarget::~AttributeMemorySaveTarget() { -} - +AttributeMemorySaveTarget::~AttributeMemorySaveTarget() = default; IAttributeFileWriter & AttributeMemorySaveTarget::datWriter() @@ -26,28 +27,24 @@ AttributeMemorySaveTarget::datWriter() return _datWriter; } - IAttributeFileWriter & AttributeMemorySaveTarget::idxWriter() { return _idxWriter; } - IAttributeFileWriter & AttributeMemorySaveTarget::weightWriter() { return _weightWriter; } - IAttributeFileWriter & AttributeMemorySaveTarget::udatWriter() { return _udatWriter; } - bool AttributeMemorySaveTarget:: writeToFile(const TuneFileAttributes &tuneFileAttributes, @@ -68,9 +65,39 @@ writeToFile(const TuneFileAttributes &tuneFileAttributes, _weightWriter.writeTo(saveTarget.weightWriter()); } } + for (const auto& entry : _writers) { + if (!saveTarget.setup_writer(entry.first, entry.second.desc)) { + return false; + } + auto& file_writer = saveTarget.get_writer(entry.first); + entry.second.writer->writeTo(file_writer); + } saveTarget.close(); return true; } +bool +AttributeMemorySaveTarget::setup_writer(const vespalib::string& file_suffix, + const vespalib::string& desc) +{ + auto writer = std::make_unique<AttributeMemoryFileWriter>(); + auto itr = _writers.find(file_suffix); + if (itr != _writers.end()) { + return false; + } + _writers.insert(std::make_pair(file_suffix, WriterEntry(std::move(writer), desc))); + return true; +} + +IAttributeFileWriter& +AttributeMemorySaveTarget::get_writer(const vespalib::string& file_suffix) +{ + auto itr = _writers.find(file_suffix); + if (itr == _writers.end()) { + throw IllegalArgumentException("File writer with suffix '" + file_suffix + "' does not exist"); + } + return *itr->second.writer; +} + } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h index f06764fa34b..9533b881099 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h +++ b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h @@ -2,11 +2,13 @@ #pragma once -#include "iattributesavetarget.h" #include "attributememoryfilewriter.h" +#include "iattributesavetarget.h" +#include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/util/rawbuf.h> +#include <vespa/vespalib/stllike/hash_fun.h> #include <memory> -#include <vespa/searchlib/common/tunefileinfo.h> +#include <unordered_map> namespace search::common { class FileHeaderContext; } @@ -16,13 +18,22 @@ class AttributeVector; /** * Class used to save an attribute vector to memory buffer(s). **/ -class AttributeMemorySaveTarget : public IAttributeSaveTarget -{ +class AttributeMemorySaveTarget : public IAttributeSaveTarget { private: + using FileWriterUP = std::unique_ptr<AttributeMemoryFileWriter>; + struct WriterEntry { + FileWriterUP writer; + vespalib::string desc; + WriterEntry(FileWriterUP writer_in, const vespalib::string& desc_in) + : writer(std::move(writer_in)), desc(desc_in) {} + }; + using WriterMap = std::unordered_map<vespalib::string, WriterEntry, vespalib::hash<vespalib::string>>; + AttributeMemoryFileWriter _datWriter; AttributeMemoryFileWriter _idxWriter; AttributeMemoryFileWriter _weightWriter; AttributeMemoryFileWriter _udatWriter; + WriterMap _writers; public: AttributeMemorySaveTarget(); @@ -40,6 +51,11 @@ public: IAttributeFileWriter &idxWriter() override; IAttributeFileWriter &weightWriter() override; IAttributeFileWriter &udatWriter() override; + + bool setup_writer(const vespalib::string& file_suffix, + const vespalib::string& desc) override; + IAttributeFileWriter& get_writer(const vespalib::string& file_suffix) override; + }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index b043bb4aaf8..ffc62d806e2 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -266,79 +266,6 @@ const IEnumStore* AttributeVector::getEnumStoreBase() const { return nullptr; } IEnumStore* AttributeVector::getEnumStoreBase() { return nullptr; } const attribute::MultiValueMappingBase * AttributeVector::getMultiValueBase() const { return nullptr; } -std::unique_ptr<FastOS_FileInterface> -AttributeVector::openFile(const char *suffix) -{ - BaseName::string fileName(getBaseFileName()); - fileName += suffix; - return FileUtil::openFile(fileName); -} - - -std::unique_ptr<FastOS_FileInterface> -AttributeVector::openDAT() -{ - return openFile(".dat"); -} - - -std::unique_ptr<FastOS_FileInterface> -AttributeVector::openIDX() -{ - return openFile(".idx"); -} - - -std::unique_ptr<FastOS_FileInterface> -AttributeVector::openWeight() -{ - return openFile(".weight"); -} - - -std::unique_ptr<FastOS_FileInterface> -AttributeVector::openUDAT() -{ - return openFile(".dat"); -} - -fileutil::LoadedBuffer::UP -AttributeVector::loadDAT() -{ - return loadFile(".dat"); -} - - -fileutil::LoadedBuffer::UP -AttributeVector::loadIDX() -{ - return loadFile(".idx"); -} - - -fileutil::LoadedBuffer::UP -AttributeVector::loadWeight() -{ - return loadFile(".weight"); -} - - -fileutil::LoadedBuffer::UP -AttributeVector::loadUDAT() -{ - return loadFile(".udat"); -} - - -fileutil::LoadedBuffer::UP -AttributeVector::loadFile(const char *suffix) -{ - BaseName::string fileName(getBaseFileName()); - fileName += suffix; - return FileUtil::loadFile(fileName); -} - - bool AttributeVector::save(vespalib::stringref fileName) { diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index a396fb70b7c..4a53f2dd5a2 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -212,11 +212,6 @@ protected: void setNumDocs(uint32_t n) { _status.setNumDocs(n); } void incNumDocs() { _status.incNumDocs(); } - LoadedBufferUP loadDAT(); - LoadedBufferUP loadIDX(); - LoadedBufferUP loadWeight(); - LoadedBufferUP loadUDAT(); - class ValueModifier { public: @@ -269,10 +264,6 @@ protected: } public: - std::unique_ptr<FastOS_FileInterface> openDAT(); - std::unique_ptr<FastOS_FileInterface> openIDX(); - std::unique_ptr<FastOS_FileInterface> openWeight(); - std::unique_ptr<FastOS_FileInterface> openUDAT(); void incGeneration(); void removeAllOldGenerations(); @@ -572,8 +563,6 @@ private: virtual bool applyWeight(DocId doc, const FieldValue &fv, const ArithmeticValueUpdate &wAdjust); virtual void onSave(IAttributeSaveTarget & saveTarget); virtual bool onLoad(); - std::unique_ptr<FastOS_FileInterface> openFile(const char *suffix); - LoadedBufferUP loadFile(const char *suffix); BaseName _baseFileName; diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp index 0304aa8f38e..59771d7ffae 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp @@ -3,6 +3,7 @@ #include "attrvector.h" #include "attrvector.hpp" #include "iattributesavetarget.h" +#include "load_utils.h" #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.attr_vector"); @@ -123,7 +124,7 @@ bool StringDirectAttribute::onLoad() setCommittedDocIdLimit(0); } - fileutil::LoadedBuffer::UP tmpBuffer(loadDAT()); + auto tmpBuffer = attribute::LoadUtils::loadDAT(*this); bool rc(tmpBuffer.get()); if (rc) { if ( ! tmpBuffer->empty()) { @@ -158,7 +159,7 @@ bool StringDirectAttribute::onLoad() } if (hasMultiValue()) { - fileutil::LoadedBuffer::UP tmpIdx(loadIDX()); + auto tmpIdx = attribute::LoadUtils::loadIDX(*this); size_t tmpIdxLen(tmpIdx->size(sizeof(uint32_t))); _idx.clear(); _idx.reserve(tmpIdxLen); diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp index cdd34725e69..4ce7575b28d 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp @@ -2,6 +2,7 @@ #pragma once #include "attrvector.h" +#include "load_utils.h" #include <vespa/vespalib/util/hdr_abort.h> #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/util/filekit.h> @@ -23,7 +24,7 @@ NumericDirectAttribute<B>::~NumericDirectAttribute() = default; template <typename B> bool NumericDirectAttribute<B>::onLoad() { - fileutil::LoadedBuffer::UP dataBuffer(B::loadDAT()); + auto dataBuffer = attribute::LoadUtils::loadDAT(*this); bool rc(dataBuffer.get()); if (rc) { const BaseType * tmpData(static_cast <const BaseType *>(dataBuffer->buffer())); @@ -56,7 +57,7 @@ bool NumericDirectAttribute<B>::onLoad() } dataBuffer.reset(); if (this->hasMultiValue()) { - fileutil::LoadedBuffer::UP idxBuffer(B::loadIDX()); + auto idxBuffer = attribute::LoadUtils::loadIDX(*this); rc = idxBuffer.get(); if (rc) { const uint32_t * tmpIdx(static_cast<const uint32_t *>(idxBuffer->buffer())); diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index 1e4bba95b4b..895e6a6f4c0 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -89,7 +89,7 @@ FlagAttributeT<B>::onLoadEnumerated(ReaderBase &attrReader) if (numValues > 0) _bitVectorSize = numDocs; - fileutil::LoadedBuffer::UP udatBuffer(this->loadUDAT()); + auto udatBuffer = attribute::LoadUtils::loadUDAT(*this); assert((udatBuffer->size() % sizeof(TT)) == 0); vespalib::ConstArrayRef<TT> map(reinterpret_cast<const TT *>(udatBuffer->buffer()), udatBuffer->size() / sizeof(TT)); diff --git a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h index 9f90544bb83..8946fc2fcdb 100644 --- a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h +++ b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h @@ -37,6 +37,19 @@ public: virtual IAttributeFileWriter &weightWriter() = 0; virtual IAttributeFileWriter &udatWriter() = 0; + /** + * Setups a custom file writer with the given file suffix and description in the file header. + * Returns false if the file writer cannot be setup or if it already exists, true otherwise. + */ + virtual bool setup_writer(const vespalib::string& file_suffix, + const vespalib::string& desc) = 0; + + /** + * Returns the file writer with the given file suffix. + * Throws vespalib::IllegalArgumentException if the file writer does not exists. + */ + virtual IAttributeFileWriter& get_writer(const vespalib::string& file_suffix) = 0; + virtual ~IAttributeSaveTarget(); }; diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp index 041daa08cd5..701c8eaf702 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp @@ -5,13 +5,81 @@ #include "loadedenumvalue.h" #include "multi_value_mapping.h" #include "multivalue.h" +#include <vespa/fastos/file.h> +#include <vespa/searchlib/util/fileutil.h> +#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/array.hpp> using search::multivalue::Value; using search::multivalue::WeightedValue; -namespace search { -namespace attribute { +namespace search::attribute { + +using FileInterfaceUP = LoadUtils::FileInterfaceUP; +using LoadedBufferUP = LoadUtils::LoadedBufferUP; + +FileInterfaceUP +LoadUtils::openFile(const AttributeVector& attr, const vespalib::string& suffix) +{ + return FileUtil::openFile(attr.getBaseFileName() + "." + suffix); +} + + + +FileInterfaceUP +LoadUtils::openDAT(const AttributeVector& attr) +{ + return openFile(attr, "dat"); +} + +FileInterfaceUP +LoadUtils::openIDX(const AttributeVector& attr) +{ + return openFile(attr, "idx"); +} + +FileInterfaceUP +LoadUtils::openWeight(const AttributeVector& attr) +{ + return openFile(attr, "weight"); +} + +bool +LoadUtils::file_exists(const AttributeVector& attr, const vespalib::string& suffix) +{ + return vespalib::fileExists(attr.getBaseFileName() + "." + suffix); +} + +LoadedBufferUP +LoadUtils::loadFile(const AttributeVector& attr, const vespalib::string& suffix) +{ + return FileUtil::loadFile(attr.getBaseFileName() + "." + suffix); +} + +LoadedBufferUP +LoadUtils::loadDAT(const AttributeVector& attr) +{ + return loadFile(attr, "dat"); +} + +LoadedBufferUP +LoadUtils::loadIDX(const AttributeVector& attr) +{ + return loadFile(attr, "idx"); +} + +LoadedBufferUP +LoadUtils::loadWeight(const AttributeVector& attr) +{ + return loadFile(attr, "weight"); +} + +LoadedBufferUP +LoadUtils::loadUDAT(const AttributeVector& attr) +{ + return loadFile(attr, "udat"); +} + #define INSTANTIATE_ARRAY(ValueType, Saver) \ template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<Value<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, Saver) @@ -40,5 +108,4 @@ INSTANTIATE_VALUE(int64_t); INSTANTIATE_VALUE(float); INSTANTIATE_VALUE(double); -} // namespace search::attribute -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.h b/searchlib/src/vespa/searchlib/attribute/load_utils.h index 050d7726ecd..cd9d98084d5 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.h +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.h @@ -6,10 +6,34 @@ #include "readerbase.h" #include <vespa/vespalib/util/arrayref.h> -namespace search { -namespace attribute { +namespace search::attribute { -/* +/** + * Helper functions used to open / load attribute vector data files from disk. + */ +class LoadUtils { +public: + using FileInterfaceUP = std::unique_ptr<FastOS_FileInterface>; + using LoadedBufferUP = std::unique_ptr<fileutil::LoadedBuffer>; + +private: + static FileInterfaceUP openFile(const AttributeVector& attr, const vespalib::string& suffix); + +public: + static FileInterfaceUP openDAT(const AttributeVector& attr); + static FileInterfaceUP openIDX(const AttributeVector& attr); + static FileInterfaceUP openWeight(const AttributeVector& attr); + + static bool file_exists(const AttributeVector& attr, const vespalib::string& suffix); + static LoadedBufferUP loadFile(const AttributeVector& attr, const vespalib::string& suffix); + + static LoadedBufferUP loadDAT(const AttributeVector& attr); + static LoadedBufferUP loadIDX(const AttributeVector& attr); + static LoadedBufferUP loadWeight(const AttributeVector& attr); + static LoadedBufferUP loadUDAT(const AttributeVector& attr); +}; + +/** * Function for loading mapping from document id to array of enum indexes * or values from enumerated attribute reader. */ @@ -20,7 +44,7 @@ loadFromEnumeratedMultiValue(MvMapping &mapping, vespalib::ConstArrayRef<typename MvMapping::MultiValueType::ValueType> enumValueToValueMap, Saver saver) __attribute((noinline)); -/* +/** * Function for loading mapping from document id to enum index or * value from enumerated attribute reader. */ @@ -32,5 +56,4 @@ loadFromEnumeratedSingleValue(Vector &vector, vespalib::ConstArrayRef<typename Vector::ValueType> enumValueToValueMap, Saver saver) __attribute((noinline)); -} // namespace search::attribute -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp index 1efa2789fcb..3ca7423c38c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp @@ -117,7 +117,7 @@ MultiValueNumericAttribute<B, M>::onLoadEnumerated(ReaderBase & attrReader) this->setCommittedDocIdLimit(numDocs); this->_mvMapping.reserve(numDocs+1); - LoadedBuffer::UP udatBuffer(this->loadUDAT()); + auto udatBuffer = attribute::LoadUtils::loadUDAT(*this); assert((udatBuffer->size() % sizeof(T)) == 0); vespalib::ConstArrayRef<T> map(reinterpret_cast<const T *>(udatBuffer->buffer()), udatBuffer->size() / sizeof(T)); uint32_t maxvc = attribute::loadFromEnumeratedMultiValue(this->_mvMapping, attrReader, map, attribute::NoSaveLoadedEnum()); diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp index 9ee365fc7cc..e17d41a5521 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp @@ -2,13 +2,14 @@ #pragma once -#include "multinumericenumattribute.h" -#include "loadednumericvalue.h" #include "attributeiterators.hpp" -#include <vespa/searchlib/util/fileutil.hpp> +#include "load_utils.h" +#include "loadednumericvalue.h" +#include "multinumericenumattribute.h" #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/queryeval/emptysearch.h> +#include <vespa/searchlib/util/fileutil.hpp> namespace search { @@ -52,7 +53,7 @@ template <typename B, typename M> bool MultiValueNumericEnumAttribute<B, M>::onLoadEnumerated(ReaderBase &attrReader) { - LoadedBuffer::UP udatBuffer(this->loadUDAT()); + auto udatBuffer = attribute::LoadUtils::loadUDAT(*this); uint32_t numDocs = attrReader.getNumIdx() - 1; uint64_t numValues = attrReader.getNumValues(); diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp index b0e4df65c2b..72cc6e38fac 100644 --- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp @@ -1,12 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "predicate_attribute.h" -#include "iattributesavetarget.h" #include "attribute_header.h" -#include <vespa/searchlib/predicate/predicate_index.h> -#include <vespa/searchlib/util/fileutil.h> +#include "iattributesavetarget.h" +#include "load_utils.h" +#include "predicate_attribute.h" #include <vespa/document/fieldvalue/predicatefieldvalue.h> #include <vespa/document/predicate/predicate.h> +#include <vespa/searchlib/predicate/predicate_index.h> +#include <vespa/searchlib/util/fileutil.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/log/log.h> @@ -183,7 +184,7 @@ struct DummyObserver : SimpleIndexDeserializeObserver<> { bool PredicateAttribute::onLoad() { - fileutil::LoadedBuffer::UP loaded_buffer = loadDAT(); + auto loaded_buffer = attribute::LoadUtils::loadDAT(*this); char *rawBuffer = const_cast<char *>(static_cast<const char *>(loaded_buffer->buffer())); size_t size = loaded_buffer->size(); DataBuffer buffer(rawBuffer, size); diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp index 62936ecaaf4..a396fe9efd8 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp @@ -1,10 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "readerbase.h" #include "attributevector.h" +#include "load_utils.h" +#include "readerbase.h" #include <vespa/fastlib/io/bufferedfile.h> -#include <vespa/vespalib/util/exceptions.h> #include <vespa/searchlib/util/filesizecalculator.h> +#include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> LOG_SETUP(".search.attribute.readerbase"); @@ -12,28 +13,27 @@ LOG_SETUP(".search.attribute.readerbase"); namespace search { namespace { - const vespalib::string versionTag = "version"; - const vespalib::string docIdLimitTag = "docIdLimit"; - const vespalib::string createSerialNumTag = "createSerialNum"; - constexpr size_t DIRECTIO_ALIGNMENT(4096); +const vespalib::string versionTag = "version"; +const vespalib::string docIdLimitTag = "docIdLimit"; +const vespalib::string createSerialNumTag = "createSerialNum"; - uint64_t - extractCreateSerialNum(const vespalib::GenericHeader &header) - { - return (header.hasTag(createSerialNumTag)) ? header.getTag(createSerialNumTag).asInteger() : 0u; - } +constexpr size_t DIRECTIO_ALIGNMENT(4096); +uint64_t +extractCreateSerialNum(const vespalib::GenericHeader &header) +{ + return (header.hasTag(createSerialNumTag)) ? header.getTag(createSerialNumTag).asInteger() : 0u; +} } ReaderBase::ReaderBase(AttributeVector &attr) - : _datFile(attr.openDAT()), + : _datFile(attribute::LoadUtils::openDAT(attr)), _weightFile(attr.hasWeightedSetType() ? - attr.openWeight() : std::unique_ptr<Fast_BufferedFile>()), + attribute::LoadUtils::openWeight(attr) : std::unique_ptr<Fast_BufferedFile>()), _idxFile(attr.hasMultiValue() ? - attr.openIDX() : std::unique_ptr<Fast_BufferedFile>()), - _udatFile(), + attribute::LoadUtils::openIDX(attr) : std::unique_ptr<Fast_BufferedFile>()), _weightReader(*_weightFile), _idxReader(*_idxFile), _enumReader(*_datFile), @@ -41,7 +41,6 @@ ReaderBase::ReaderBase(AttributeVector &attr) _datHeaderLen(0u), _idxHeaderLen(0u), _weightHeaderLen(0u), - _udatHeaderLen(0u), _createSerialNum(0u), _fixedWidth(attr.getFixedWidth()), _enumerated(false), @@ -83,20 +82,12 @@ ReaderBase::ReaderBase(AttributeVector &attr) } if (hasData() && AttributeVector::isEnumerated(_datHeader)) { _enumerated = true; - _udatFile = attr.openUDAT(); - vespalib::FileHeader udatHeader(DIRECTIO_ALIGNMENT); - _udatHeaderLen = udatHeader.readFile(*_udatFile); - _udatFile->SetPosition(_udatHeaderLen); - if (!attr.headerTypeOK(udatHeader)) - _udatFile->Close(); } _hasLoadData = hasData() && (!attr.hasMultiValue() || hasIdx()) && - (!attr.hasWeightedSetType() || hasWeight()) && - (!getEnumerated() || hasUData()); + (!attr.hasWeightedSetType() || hasWeight()); } - ReaderBase::~ReaderBase() = default; bool @@ -115,11 +106,6 @@ ReaderBase::hasData() const { } bool -ReaderBase::hasUData() const { - return _udatFile.get() && _udatFile->IsOpened(); -} - -bool ReaderBase:: extractFileSize(const vespalib::GenericHeader &header, FastOS_FileInterface &file, uint64_t &fileSize) @@ -129,7 +115,6 @@ extractFileSize(const vespalib::GenericHeader &header, file.GetFileName(), fileSize); } - void ReaderBase::rewind() { @@ -142,12 +127,8 @@ ReaderBase::rewind() if (hasWeight()) { _weightFile->SetPosition(_weightHeaderLen); } - if (getEnumerated()) { - _udatFile->SetPosition(_udatHeaderLen); - } } - size_t ReaderBase::getNumValues() { @@ -169,7 +150,6 @@ ReaderBase::getNumValues() } } - uint32_t ReaderBase::getNextValueCount() { diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.h b/searchlib/src/vespa/searchlib/attribute/readerbase.h index 09db52f5e25..a7685e4532a 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.h @@ -19,7 +19,6 @@ public: bool hasWeight() const; bool hasIdx() const; bool hasData() const; - bool hasUData() const; uint32_t getNumIdx() const { return (_idxFileSize - _idxHeaderLen) /sizeof(uint32_t); @@ -51,7 +50,6 @@ protected: private: std::unique_ptr<FastOS_FileInterface> _weightFile; std::unique_ptr<FastOS_FileInterface> _idxFile; - std::unique_ptr<FastOS_FileInterface> _udatFile; FileReader<int32_t> _weightReader; FileReader<uint32_t> _idxReader; FileReader<uint32_t> _enumReader; @@ -59,7 +57,6 @@ private: uint32_t _datHeaderLen; uint32_t _idxHeaderLen; uint32_t _weightHeaderLen; - uint32_t _udatHeaderLen; uint64_t _createSerialNum; size_t _fixedWidth; bool _enumerated; diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index b055af7c084..9421730f335 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attributesaver.h" +#include "load_utils.h" #include "readerbase.h" #include "reference_attribute.h" #include "reference_attribute_saver.h" @@ -223,7 +224,7 @@ ReferenceAttribute::onLoad() uint64_t numValues(0); numValues = attrReader.getEnumCount(); numDocs = numValues; - fileutil::LoadedBuffer::UP udatBuffer(loadUDAT()); + auto udatBuffer = attribute::LoadUtils::loadUDAT(*this); const GenericHeader &header = udatBuffer->getHeader(); uint32_t uniqueValueCount = extractUniqueValueCount(header); assert(uniqueValueCount * sizeof(GlobalId) == udatBuffer->size()); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index 69d4e6a5ee9..681c2af1f07 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "singlenumericattribute.h" +#include "attributeiterators.hpp" #include "attributevector.hpp" -#include "singlenumericattributesaver.h" #include "load_utils.h" #include "primitivereader.h" -#include "attributeiterators.hpp" +#include "singlenumericattribute.h" +#include "singlenumericattributesaver.h" #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/queryeval/emptysearch.h> @@ -114,7 +114,7 @@ SingleValueNumericAttribute<B>::onLoadEnumerated(ReaderBase &attrReader) this->setCommittedDocIdLimit(numDocs); _data.unsafe_reserve(numDocs); - fileutil::LoadedBuffer::UP udatBuffer(this->loadUDAT()); + auto udatBuffer = attribute::LoadUtils::loadUDAT(*this); assert((udatBuffer->size() % sizeof(T)) == 0); vespalib::ConstArrayRef<T> map(reinterpret_cast<const T *>(udatBuffer->buffer()), udatBuffer->size() / sizeof(T)); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp index 990388d2a12..5fb587c908e 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp @@ -2,14 +2,15 @@ #pragma once -#include "singlenumericenumattribute.h" -#include <vespa/searchlib/common/sort.h> -#include "singleenumattribute.hpp" +#include "attributeiterators.hpp" +#include "load_utils.h" #include "loadednumericvalue.h" #include "primitivereader.h" -#include "attributeiterators.hpp" -#include <vespa/searchlib/queryeval/emptysearch.h> +#include "singleenumattribute.hpp" +#include "singlenumericenumattribute.h" +#include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/query/query_term_simple.h> +#include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/util/fileutil.hpp> namespace search { @@ -79,7 +80,7 @@ template <typename B> bool SingleValueNumericEnumAttribute<B>::onLoadEnumerated(ReaderBase &attrReader) { - fileutil::LoadedBuffer::UP udatBuffer(this->loadUDAT()); + auto udatBuffer = attribute::LoadUtils::loadUDAT(*this); uint64_t numValues = attrReader.getEnumCount(); uint32_t numDocs = numValues; diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index 32b5b3ca373..40e706e924d 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -1,11 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "stringbase.h" #include "attributevector.hpp" +#include "load_utils.h" #include "readerbase.h" +#include "stringbase.h" #include <vespa/document/fieldvalue/fieldvalue.h> -#include <vespa/searchlib/util/fileutil.hpp> #include <vespa/searchlib/query/query_term_ucs4.h> +#include <vespa/searchlib/util/fileutil.hpp> #include <vespa/vespalib/locale/c.h> #include <vespa/vespalib/util/array.hpp> @@ -316,7 +317,7 @@ bool StringAttribute::apply(DocId, const ArithmeticValueUpdate & ) bool StringAttribute::onLoadEnumerated(ReaderBase &attrReader) { - fileutil::LoadedBuffer::UP udatBuffer(loadUDAT()); + auto udatBuffer = attribute::LoadUtils::loadUDAT(*this); bool hasIdx(attrReader.hasIdx()); size_t numDocs(0); diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt index 2ee722902c8..f4a9e27b79d 100644 --- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_common OBJECT SOURCES + adaptive_sequenced_executor.cpp allocatedbitvector.cpp bitvector.cpp bitvectorcache.cpp diff --git a/searchlib/src/vespa/searchlib/common/adaptive_sequenced_executor.cpp b/searchlib/src/vespa/searchlib/common/adaptive_sequenced_executor.cpp new file mode 100644 index 00000000000..f31172b1eba --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/adaptive_sequenced_executor.cpp @@ -0,0 +1,324 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "adaptive_sequenced_executor.h" + +namespace search { + +//----------------------------------------------------------------------------- + +AdaptiveSequencedExecutor::Strand::Strand() + : state(State::IDLE), + queue() +{ +} + +AdaptiveSequencedExecutor::Strand::~Strand() +{ + assert(queue.empty()); +} + +//----------------------------------------------------------------------------- + +AdaptiveSequencedExecutor::Worker::Worker() + : cond(), + state(State::RUNNING), + strand(nullptr) +{ +} + +AdaptiveSequencedExecutor::Worker::~Worker() +{ + assert(state == State::DONE); + assert(strand == nullptr); +} + +//----------------------------------------------------------------------------- + +AdaptiveSequencedExecutor::Self::Self() + : cond(), + state(State::OPEN), + waiting_tasks(0), + pending_tasks(0) +{ +} + +AdaptiveSequencedExecutor::Self::~Self() +{ + assert(state == State::CLOSED); + assert(waiting_tasks == 0); + assert(pending_tasks == 0); +} + +//----------------------------------------------------------------------------- + +AdaptiveSequencedExecutor::ThreadTools::ThreadTools(AdaptiveSequencedExecutor &parent_in) + : parent(parent_in), + pool(std::make_unique<FastOS_ThreadPool>(STACK_SIZE)), + allow_worker_exit() +{ +} + +AdaptiveSequencedExecutor::ThreadTools::~ThreadTools() +{ + assert(pool->isClosed()); +} + +void +AdaptiveSequencedExecutor::ThreadTools::Run(FastOS_ThreadInterface *, void *) +{ + parent.worker_main(); +} + +void +AdaptiveSequencedExecutor::ThreadTools::start(size_t num_threads) +{ + for (size_t i = 0; i < num_threads; ++i) { + FastOS_ThreadInterface *thread = pool->NewThread(this); + assert(thread != nullptr); + (void)thread; + } +} + +void +AdaptiveSequencedExecutor::ThreadTools::close() +{ + allow_worker_exit.countDown(); + pool->Close(); +} + +//----------------------------------------------------------------------------- + +void +AdaptiveSequencedExecutor::maybe_block_self(std::unique_lock<std::mutex> &lock) +{ + while (_self.state == Self::State::BLOCKED) { + _self.cond.wait(lock); + } + while ((_self.state == Self::State::OPEN) && (_self.pending_tasks >= _cfg.max_pending)) { + _self.state = Self::State::BLOCKED; + while (_self.state == Self::State::BLOCKED) { + _self.cond.wait(lock); + } + } +} + +bool +AdaptiveSequencedExecutor::maybe_unblock_self(const std::unique_lock<std::mutex> &) +{ + if ((_self.state == Self::State::BLOCKED) && (_self.pending_tasks < _cfg.wakeup_limit)) { + _self.state = Self::State::OPEN; + return true; + } + return false; +} + +AdaptiveSequencedExecutor::Worker * +AdaptiveSequencedExecutor::get_worker_to_wake(const std::unique_lock<std::mutex> &) +{ + if ((_self.waiting_tasks > _cfg.max_waiting) && (!_worker_stack.empty())) { + assert(!_wait_queue.empty()); + Worker *worker = _worker_stack.back(); + _worker_stack.popBack(); + assert(worker->state == Worker::State::BLOCKED); + assert(worker->strand == nullptr); + worker->state = Worker::State::RUNNING; + worker->strand = _wait_queue.front(); + _wait_queue.pop(); + assert(worker->strand->state == Strand::State::WAITING); + assert(!worker->strand->queue.empty()); + worker->strand->state = Strand::State::ACTIVE; + assert(_self.waiting_tasks >= worker->strand->queue.size()); + _self.waiting_tasks -= worker->strand->queue.size(); + return worker; + } + return nullptr; +} + +bool +AdaptiveSequencedExecutor::obtain_strand(Worker &worker, std::unique_lock<std::mutex> &lock) +{ + assert(worker.strand == nullptr); + if (!_wait_queue.empty()) { + worker.strand = _wait_queue.front(); + _wait_queue.pop(); + assert(worker.strand->state == Strand::State::WAITING); + assert(!worker.strand->queue.empty()); + worker.strand->state = Strand::State::ACTIVE; + assert(_self.waiting_tasks >= worker.strand->queue.size()); + _self.waiting_tasks -= worker.strand->queue.size(); + } else if (_self.state == Self::State::CLOSED) { + worker.state = Worker::State::DONE; + } else { + worker.state = Worker::State::BLOCKED; + _worker_stack.push(&worker); + while (worker.state == Worker::State::BLOCKED) { + worker.cond.wait(lock); + } + } + return (worker.state == Worker::State::RUNNING); +} + +bool +AdaptiveSequencedExecutor::exchange_strand(Worker &worker, std::unique_lock<std::mutex> &lock) +{ + if (worker.strand == nullptr) { + return obtain_strand(worker, lock); + } + if (worker.strand->queue.empty()) { + worker.strand->state = Strand::State::IDLE; + worker.strand = nullptr; + return obtain_strand(worker, lock); + } + if (!_wait_queue.empty()) { + worker.strand->state = Strand::State::WAITING; + _self.waiting_tasks += worker.strand->queue.size(); + _wait_queue.push(worker.strand); + worker.strand = nullptr; + return obtain_strand(worker, lock); + } + return true; +} + +AdaptiveSequencedExecutor::Task::UP +AdaptiveSequencedExecutor::next_task(Worker &worker) +{ + Task::UP task; + Worker *worker_to_wake = nullptr; + auto guard = std::unique_lock(_mutex); + if (exchange_strand(worker, guard)) { + assert(worker.state == Worker::State::RUNNING); + assert(worker.strand != nullptr); + assert(!worker.strand->queue.empty()); + task = std::move(worker.strand->queue.front()); + worker.strand->queue.pop(); + _stats.queueSize.add(--_self.pending_tasks); + worker_to_wake = get_worker_to_wake(guard); + } else { + assert(worker.state == Worker::State::DONE); + assert(worker.strand == nullptr); + } + bool signal_self = maybe_unblock_self(guard); + guard.unlock(); // UNLOCK + if (worker_to_wake != nullptr) { + worker_to_wake->cond.notify_one(); + } + if (signal_self) { + _self.cond.notify_all(); + } + return task; +} + +void +AdaptiveSequencedExecutor::worker_main() +{ + Worker worker; + while (Task::UP my_task = next_task(worker)) { + my_task->run(); + } + _thread_tools->allow_worker_exit.await(); +} + +AdaptiveSequencedExecutor::AdaptiveSequencedExecutor(size_t num_strands, size_t num_threads, + size_t max_waiting, size_t max_pending) + : ISequencedTaskExecutor(num_strands), + _thread_tools(std::make_unique<ThreadTools>(*this)), + _mutex(), + _strands(num_strands), + _wait_queue(num_strands), + _worker_stack(num_threads), + _self(), + _stats(), + _cfg(num_threads, max_waiting, max_pending) +{ + _stats.queueSize.add(_self.pending_tasks); + _thread_tools->start(num_threads); +} + +AdaptiveSequencedExecutor::~AdaptiveSequencedExecutor() +{ + sync(); + { + auto guard = std::unique_lock(_mutex); + assert(_self.state == Self::State::OPEN); + _self.state = Self::State::CLOSED; + while (!_worker_stack.empty()) { + Worker *worker = _worker_stack.back(); + _worker_stack.popBack(); + assert(worker->state == Worker::State::BLOCKED); + assert(worker->strand == nullptr); + worker->state = Worker::State::DONE; + worker->cond.notify_one(); + } + _self.cond.notify_all(); + } + _thread_tools->close(); + assert(_wait_queue.empty()); + assert(_worker_stack.empty()); +} + +void +AdaptiveSequencedExecutor::executeTask(ExecutorId id, Task::UP task) +{ + assert(id.getId() < _strands.size()); + Strand &strand = _strands[id.getId()]; + auto guard = std::unique_lock(_mutex); + maybe_block_self(guard); + assert(_self.state != Self::State::CLOSED); + strand.queue.push(std::move(task)); + _stats.queueSize.add(++_self.pending_tasks); + ++_stats.acceptedTasks; + if (strand.state == Strand::State::WAITING) { + ++_self.waiting_tasks; + } else if (strand.state == Strand::State::IDLE) { + if (_worker_stack.size() < _cfg.num_threads) { + strand.state = Strand::State::WAITING; + _wait_queue.push(&strand); + _self.waiting_tasks += strand.queue.size(); + } else { + strand.state = Strand::State::ACTIVE; + assert(_wait_queue.empty()); + Worker *worker = _worker_stack.back(); + _worker_stack.popBack(); + assert(worker->state == Worker::State::BLOCKED); + assert(worker->strand == nullptr); + worker->state = Worker::State::RUNNING; + worker->strand = &strand; + guard.unlock(); // UNLOCK + worker->cond.notify_one(); + } + } +} + +void +AdaptiveSequencedExecutor::sync() +{ + vespalib::CountDownLatch latch(_strands.size()); + for (size_t i = 0; i < _strands.size(); ++i) { + execute(ExecutorId(i), [&](){ latch.countDown(); }); + } + latch.await(); +} + +void +AdaptiveSequencedExecutor::setTaskLimit(uint32_t task_limit) +{ + auto guard = std::unique_lock(_mutex); + _cfg.set_max_pending(task_limit); + bool signal_self = maybe_unblock_self(guard); + guard.unlock(); // UNLOCK + if (signal_self) { + _self.cond.notify_all(); + } +} + +AdaptiveSequencedExecutor::Stats +AdaptiveSequencedExecutor::getStats() +{ + auto guard = std::lock_guard(_mutex); + Stats stats = _stats; + _stats = Stats(); + _stats.queueSize.add(_self.pending_tasks); + return stats; +} + +} diff --git a/searchlib/src/vespa/searchlib/common/adaptive_sequenced_executor.h b/searchlib/src/vespa/searchlib/common/adaptive_sequenced_executor.h new file mode 100644 index 00000000000..3abc095e9df --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/adaptive_sequenced_executor.h @@ -0,0 +1,126 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "isequencedtaskexecutor.h" +#include <vespa/vespalib/util/arrayqueue.hpp> +#include <vespa/vespalib/util/gate.h> +#include <vespa/fastos/thread.h> +#include <mutex> +#include <condition_variable> +#include <cassert> + +namespace search { + +/** + * Sequenced executor that balances the number of active threads in + * order to optimize for throughput over latency by minimizing the + * number of critical-path wakeups. + **/ +class AdaptiveSequencedExecutor : public ISequencedTaskExecutor +{ +private: + using Stats = vespalib::ExecutorStats; + using Task = vespalib::Executor::Task; + + /** + * Values used to configure the executor. + **/ + struct Config { + size_t num_threads; + size_t max_waiting; + size_t max_pending; + size_t wakeup_limit; + void set_max_pending(size_t max_pending_in) { + max_pending = std::max(1uL, max_pending_in); + wakeup_limit = std::max(1uL, size_t(max_pending * 0.9)); + assert(wakeup_limit > 0); + assert(wakeup_limit <= max_pending); + } + Config(size_t num_threads_in, size_t max_waiting_in, size_t max_pending_in) + : num_threads(num_threads_in), max_waiting(max_waiting_in), max_pending(1000), wakeup_limit(900) + { + assert(num_threads > 0); + set_max_pending(max_pending_in); + } + }; + + /** + * Tasks that need to be sequenced are handled by a single strand. + **/ + struct Strand { + enum class State { IDLE, WAITING, ACTIVE }; + State state; + vespalib::ArrayQueue<Task::UP> queue; + Strand(); + ~Strand(); + }; + + /** + * The state of a single worker thread. + **/ + struct Worker { + enum class State { RUNNING, BLOCKED, DONE }; + std::condition_variable cond; + State state; + Strand *strand; + Worker(); + ~Worker(); + }; + + /** + * State related to the executor itself. + **/ + struct Self { + enum class State { OPEN, BLOCKED, CLOSED }; + std::condition_variable cond; + State state; + size_t waiting_tasks; + size_t pending_tasks; + Self(); + ~Self(); + }; + + /** + * Stuff related to worker thread startup and shutdown. + **/ + struct ThreadTools : FastOS_Runnable { + static constexpr size_t STACK_SIZE = (256 * 1024); + AdaptiveSequencedExecutor &parent; + std::unique_ptr<FastOS_ThreadPool> pool; + vespalib::Gate allow_worker_exit; + ThreadTools(AdaptiveSequencedExecutor &parent_in); + ~ThreadTools(); + void Run(FastOS_ThreadInterface *, void *) override; + void start(size_t num_threads); + void close(); + }; + + std::unique_ptr<ThreadTools> _thread_tools; + std::mutex _mutex; + std::vector<Strand> _strands; + vespalib::ArrayQueue<Strand*> _wait_queue; + vespalib::ArrayQueue<Worker*> _worker_stack; + Self _self; + Stats _stats; + Config _cfg; + + void maybe_block_self(std::unique_lock<std::mutex> &lock); + bool maybe_unblock_self(const std::unique_lock<std::mutex> &lock); + + Worker *get_worker_to_wake(const std::unique_lock<std::mutex> &lock); + bool obtain_strand(Worker &worker, std::unique_lock<std::mutex> &lock); + bool exchange_strand(Worker &worker, std::unique_lock<std::mutex> &lock); + Task::UP next_task(Worker &worker); + void worker_main(); +public: + AdaptiveSequencedExecutor(size_t num_strands, size_t num_threads, + size_t max_waiting, size_t max_pending); + ~AdaptiveSequencedExecutor() override; + void executeTask(ExecutorId id, Task::UP task) override; + void sync() override; + void setTaskLimit(uint32_t task_limit) override; + vespalib::ExecutorStats getStats() override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp index 4c501defeea..a93eb1ff4bc 100644 --- a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp +++ b/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp @@ -39,7 +39,7 @@ void ForegroundTaskExecutor::setTaskLimit(uint32_t) { } vespalib::ExecutorStats ForegroundTaskExecutor::getStats() { - return vespalib::ExecutorStats(0, _accepted.load(std::memory_order_relaxed), 0); + return vespalib::ExecutorStats(vespalib::ExecutorStats::QueueSizeT(0) , _accepted.load(std::memory_order_relaxed), 0); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index ec31bcb5117..a8737a19eec 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -224,25 +224,27 @@ private: template <typename A> class SingleDotProductExecutorByValue final : public fef::FeatureExecutor { public: - SingleDotProductExecutorByValue(const A * attribute, multivalue::WeightedValue<typename A::BaseType> keyValue) + SingleDotProductExecutorByValue(const A * attribute, typename A::BaseType key, feature_t value) : _attribute(attribute), - _keyValue(keyValue) + _key(key), + _value(value) {} void execute(uint32_t docId) override { const multivalue::WeightedValue<typename A::BaseType> *values(nullptr); uint32_t sz = _attribute->getRawValues(docId, values); for (size_t i = 0; i < sz; ++i) { - if (values[i].value() == _keyValue.value()) { - outputs().set_number(0, values[i].weight()*_keyValue.weight()); + if (values[i].value() == _key) { + outputs().set_number(0, values[i].weight() * _value); return; } } outputs().set_number(0, 0); } private: - const A * _attribute; - multivalue::WeightedValue<typename A::BaseType> _keyValue; + const A * _attribute; + typename A::BaseType _key; + feature_t _value; }; } @@ -628,9 +630,9 @@ size_t extractSize(const dotproduct::wset::IntegerVectorT<T> & v) { } template<typename T> -multivalue::WeightedValue<T> extractElem(const dotproduct::wset::IntegerVectorT<T> & v, size_t idx) { +std::pair<T, feature_t> extractElem(const dotproduct::wset::IntegerVectorT<T> & v, size_t idx) { const auto & pair = v.getVector()[idx]; - return multivalue::WeightedValue<T>(pair.first, pair.second); + return std::pair<T, feature_t>(pair.first, pair.second); } template<typename T> @@ -639,7 +641,7 @@ size_t extractSize(const std::unique_ptr<dotproduct::wset::IntegerVectorT<T>> & } template<typename T> -multivalue::WeightedValue<T> extractElem(const std::unique_ptr<dotproduct::wset::IntegerVectorT<T>> & v, size_t idx) { +std::pair<T, feature_t> extractElem(const std::unique_ptr<dotproduct::wset::IntegerVectorT<T>> & v, size_t idx) { return extractElem(*v, idx); } @@ -656,7 +658,8 @@ createForDirectWSetImpl(const IAttributeVector * attribute, V && vector, vespali auto * exactA = dynamic_cast<const ExactA *>(iattr); if (exactA != nullptr) { if (extractSize(vector) == 1) { - return stash.create<SingleDotProductExecutorByValue<ExactA>>(exactA, extractElem(vector, 0ul)); + auto elem = extractElem(vector, 0ul); + return stash.create<SingleDotProductExecutorByValue<ExactA>>(exactA, elem.first, elem.second); } return stash.create<DotProductExecutor<ExactA>>(exactA, std::forward<V>(vector)); } diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 7090158c773..0f106f693f8 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -9,11 +9,15 @@ vespa_add_library(searchlib_tensor OBJECT generic_tensor_attribute.cpp generic_tensor_attribute_saver.cpp generic_tensor_store.cpp + hnsw_graph.cpp hnsw_index.cpp + hnsw_index_loader.cpp + hnsw_index_saver.cpp imported_tensor_attribute_vector.cpp imported_tensor_attribute_vector_read_guard.cpp inv_log_level_generator.cpp nearest_neighbor_index.cpp + nearest_neighbor_index_saver.cpp tensor_attribute.cpp tensor_store.cpp DEPENDS diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 627f7f0dfa9..68ce0c1bb00 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -3,16 +3,19 @@ #include "dense_tensor_attribute.h" #include "dense_tensor_attribute_saver.h" #include "nearest_neighbor_index.h" +#include "nearest_neighbor_index_saver.h" #include "tensor_attribute.hpp" #include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h> #include <vespa/eval/tensor/tensor.h> #include <vespa/fastlib/io/bufferedfile.h> +#include <vespa/searchlib/attribute/load_utils.h> #include <vespa/searchlib/attribute/readerbase.h> #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.tensor.dense_tensor_attribute"); +using search::attribute::LoadUtils; using vespalib::eval::ValueType; using vespalib::slime::ObjectInserter; using vespalib::tensor::MutableDenseTensorView; @@ -148,6 +151,8 @@ DenseTensorAttribute::onLoad() if (!tensorReader.hasData()) { return false; } + bool has_index_file = LoadUtils::file_exists(*this, DenseTensorAttributeSaver::index_file_suffix()); + setCreateSerialNum(tensorReader.getCreateSerialNum()); assert(tensorReader.getVersion() == DENSE_TENSOR_ATTRIBUTE_VERSION); assert(getConfig().tensorType().to_spec() == @@ -160,7 +165,7 @@ DenseTensorAttribute::onLoad() auto raw = _denseTensorStore.allocRawBuffer(); tensorReader.readTensor(raw.data, _denseTensorStore.getBufSize()); _refVector.push_back(raw.ref); - if (_index) { + if (_index && !has_index_file) { // This ensures that get_vector() (via getTensor()) is able to find the newly added tensor. setCommittedDocIdLimit(lid + 1); _index->add_document(lid); @@ -171,6 +176,12 @@ DenseTensorAttribute::onLoad() } setNumDocs(numDocs); setCommittedDocIdLimit(numDocs); + if (_index && has_index_file) { + auto buffer = LoadUtils::loadFile(*this, DenseTensorAttributeSaver::index_file_suffix()); + if (!_index->load(*buffer)) { + return false; + } + } return true; } @@ -180,11 +191,13 @@ DenseTensorAttribute::onInitSave(vespalib::stringref fileName) { vespalib::GenerationHandler::Guard guard(getGenerationHandler(). takeGuard()); + auto index_saver = (_index ? _index->make_saver() : std::unique_ptr<NearestNeighborIndexSaver>()); return std::make_unique<DenseTensorAttributeSaver> (std::move(guard), this->createAttributeHeader(fileName), getRefCopy(), - _denseTensorStore); + _denseTensorStore, + std::move(index_saver)); } void diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp index d78adab81b5..fd8d6162f01 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp @@ -1,20 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dense_tensor_attribute_saver.h" -#include <vespa/vespalib/util/bufferwriter.h> #include "dense_tensor_store.h" +#include "nearest_neighbor_index_saver.h" +#include <vespa/vespalib/util/bufferwriter.h> #include <vespa/searchlib/attribute/iattributesavetarget.h> using vespalib::GenerationHandler; -namespace search { - -namespace tensor { +namespace search::tensor { namespace { -static const uint8_t tensorIsNotPresent = 0; -static const uint8_t tensorIsPresent = 1; +constexpr uint8_t tensorIsNotPresent = 0; +constexpr uint8_t tensorIsPresent = 1; } @@ -22,42 +21,60 @@ DenseTensorAttributeSaver:: DenseTensorAttributeSaver(GenerationHandler::Guard &&guard, const attribute::AttributeHeader &header, RefCopyVector &&refs, - const DenseTensorStore &tensorStore) + const DenseTensorStore &tensorStore, + IndexSaverUP index_saver) : AttributeSaver(std::move(guard), header), _refs(std::move(refs)), - _tensorStore(tensorStore) + _tensorStore(tensorStore), + _index_saver(std::move(index_saver)) { } +DenseTensorAttributeSaver::~DenseTensorAttributeSaver() = default; -DenseTensorAttributeSaver::~DenseTensorAttributeSaver() +vespalib::string +DenseTensorAttributeSaver::index_file_suffix() { + return "nnidx"; } - bool DenseTensorAttributeSaver::onSave(IAttributeSaveTarget &saveTarget) { - std::unique_ptr<BufferWriter> - datWriter(saveTarget.datWriter().allocBufferWriter()); + if (_index_saver) { + if (!saveTarget.setup_writer(index_file_suffix(), "Binary data file for nearest neighbor index")) { + return false; + } + } + + auto dat_writer = saveTarget.datWriter().allocBufferWriter(); + save_tensor_store(*dat_writer); + + if (_index_saver) { + auto index_writer = saveTarget.get_writer(index_file_suffix()).allocBufferWriter(); + // Note: Implementation of save() is responsible to call BufferWriter::flush(). + _index_saver->save(*index_writer); + } + return true; +} + +void +DenseTensorAttributeSaver::save_tensor_store(BufferWriter& writer) const +{ const uint32_t docIdLimit(_refs.size()); const uint32_t cellSize = _tensorStore.getCellSize(); for (uint32_t lid = 0; lid < docIdLimit; ++lid) { if (_refs[lid].valid()) { auto raw = _tensorStore.getRawBuffer(_refs[lid]); - datWriter->write(&tensorIsPresent, sizeof(tensorIsPresent)); + writer.write(&tensorIsPresent, sizeof(tensorIsPresent)); size_t numCells = _tensorStore.getNumCells(); size_t rawLen = numCells * cellSize; - datWriter->write(static_cast<const char *>(raw), rawLen); + writer.write(static_cast<const char *>(raw), rawLen); } else { - datWriter->write(&tensorIsNotPresent, sizeof(tensorIsNotPresent)); + writer.write(&tensorIsNotPresent, sizeof(tensorIsNotPresent)); } } - datWriter->flush(); - return true; + writer.flush(); } - -} // namespace search::tensor - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h index 1f6596e82f5..895e2951cea 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h @@ -5,28 +5,41 @@ #include "tensor_attribute.h" #include <vespa/searchlib/attribute/attributesaver.h> +namespace search { class BufferWriter; } + namespace search::tensor { class DenseTensorStore; +class NearestNeighborIndexSaver; -/* - * Class for saving a tensor attribute. +/** + * Class for saving a dense tensor attribute. + * Will also save the nearest neighbor index if existing. */ -class DenseTensorAttributeSaver : public AttributeSaver -{ +class DenseTensorAttributeSaver : public AttributeSaver { public: using RefCopyVector = TensorAttribute::RefCopyVector; private: + using GenerationHandler = vespalib::GenerationHandler; + using IndexSaverUP = std::unique_ptr<NearestNeighborIndexSaver>; + RefCopyVector _refs; const DenseTensorStore &_tensorStore; - using GenerationHandler = vespalib::GenerationHandler; + IndexSaverUP _index_saver; bool onSave(IAttributeSaveTarget &saveTarget) override; + void save_tensor_store(BufferWriter& writer) const; + public: - DenseTensorAttributeSaver(GenerationHandler::Guard &&guard, const attribute::AttributeHeader &header, - RefCopyVector &&refs, const DenseTensorStore &tensorStore); + DenseTensorAttributeSaver(GenerationHandler::Guard &&guard, + const attribute::AttributeHeader &header, + RefCopyVector &&refs, + const DenseTensorStore &tensorStore, + IndexSaverUP index_saver); ~DenseTensorAttributeSaver() override; + + static vespalib::string index_file_suffix(); }; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp new file mode 100644 index 00000000000..6f902a30861 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp @@ -0,0 +1,72 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "hnsw_graph.h" +#include "hnsw_index.h" +#include <vespa/vespalib/datastore/array_store.hpp> +#include <vespa/vespalib/util/rcuvector.hpp> + +namespace search::tensor { + +HnswGraph::HnswGraph() + : node_refs(), + nodes(HnswIndex::make_default_node_store_config()), + links(HnswIndex::make_default_link_store_config()), + entry_docid(0), // Note that docid 0 is reserved and never used + entry_level(-1) +{} + +HnswGraph::~HnswGraph() {} + +void +HnswGraph::make_node_for_document(uint32_t docid, uint32_t num_levels) +{ + node_refs.ensure_size(docid + 1, AtomicEntryRef()); + // A document cannot be added twice. + assert(!node_refs[docid].load_acquire().valid()); + // Note: The level array instance lives as long as the document is present in the index. + vespalib::Array<AtomicEntryRef> levels(num_levels, AtomicEntryRef()); + auto node_ref = nodes.add(levels); + node_refs[docid].store_release(node_ref); +} + +void +HnswGraph::remove_node_for_document(uint32_t docid) +{ + auto node_ref = node_refs[docid].load_acquire(); + nodes.remove(node_ref); + search::datastore::EntryRef invalid; + node_refs[docid].store_release(invalid); +} + +void +HnswGraph::set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& new_links) +{ + auto new_links_ref = links.add(new_links); + auto node_ref = node_refs[docid].load_acquire(); + assert(node_ref.valid()); + auto levels = nodes.get_writable(node_ref); + auto old_links_ref = levels[level].load_acquire(); + levels[level].store_release(new_links_ref); + links.remove(old_links_ref); +} + +std::vector<uint32_t> +HnswGraph::level_histogram() const +{ + std::vector<uint32_t> result; + size_t num_nodes = node_refs.size(); + for (size_t i = 0; i < num_nodes; ++i) { + uint32_t levels = 0; + auto node_ref = node_refs[i].load_acquire(); + if (node_ref.valid()) { + levels = nodes.get(node_ref).size(); + } + while (result.size() <= levels) { + result.push_back(0); + } + ++result[levels]; + } + return result; +} + +} // namespace diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h new file mode 100644 index 00000000000..233b9087af7 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h @@ -0,0 +1,76 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/datastore/array_store.h> +#include <vespa/vespalib/datastore/atomic_entry_ref.h> +#include <vespa/vespalib/datastore/entryref.h> +#include <vespa/vespalib/util/rcuvector.h> + +namespace search::tensor { + +/** + * Stroage of a hierarchical navigable small world graph (HNSW) + * that is used for approximate K-nearest neighbor search. + */ +struct HnswGraph { + using AtomicEntryRef = search::datastore::AtomicEntryRef; + + // This uses 10 bits for buffer id -> 1024 buffers. + // As we have very short arrays we get less fragmentation with fewer and larger buffers. + using EntryRefType = search::datastore::EntryRefT<22>; + + // Provides mapping from document id -> node reference. + // The reference is used to lookup the node data in NodeStore. + using NodeRefVector = vespalib::RcuVector<AtomicEntryRef>; + + // This stores the level arrays for all nodes. + // Each node consists of an array of levels (from level 0 to n) where each entry is a reference to the link array at that level. + using NodeStore = search::datastore::ArrayStore<AtomicEntryRef, EntryRefType>; + using StoreConfig = search::datastore::ArrayStoreConfig; + using LevelArrayRef = NodeStore::ConstArrayRef; + + // This stores all link arrays. + // A link array consists of the document ids of the nodes a particular node is linked to. + using LinkStore = search::datastore::ArrayStore<uint32_t, EntryRefType>; + using LinkArrayRef = LinkStore::ConstArrayRef; + + NodeRefVector node_refs; + NodeStore nodes; + LinkStore links; + uint32_t entry_docid; + int32_t entry_level; + + HnswGraph(); + + ~HnswGraph(); + + void make_node_for_document(uint32_t docid, uint32_t num_levels); + + void remove_node_for_document(uint32_t docid); + + LevelArrayRef get_level_array(uint32_t docid) const { + auto node_ref = node_refs[docid].load_acquire(); + assert(node_ref.valid()); + return nodes.get(node_ref); + } + + LinkArrayRef get_link_array(uint32_t docid, uint32_t level) const { + auto levels = get_level_array(docid); + assert(level < levels.size()); + return links.get(levels[level].load_acquire()); + } + + void set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& new_links); + + void set_entry_node(uint32_t docid, int32_t level) { + entry_docid = docid; + entry_level = level; + } + + size_t size() const { return node_refs.size(); } + + std::vector<uint32_t> level_histogram() const; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 988264c0455..b08d862ae6d 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -2,6 +2,8 @@ #include "distance_function.h" #include "hnsw_index.h" +#include "hnsw_index_loader.h" +#include "hnsw_index_saver.h" #include "random_level_generator.h" #include <vespa/searchlib/util/state_explorer_utils.h> #include <vespa/eval/tensor/dense/typed_cells.h> @@ -66,54 +68,6 @@ HnswIndex::max_links_for_level(uint32_t level) const return (level == 0) ? _cfg.max_links_at_level_0() : _cfg.max_links_on_inserts(); } -void -HnswIndex::make_node_for_document(uint32_t docid, uint32_t num_levels) -{ - _node_refs.ensure_size(docid + 1, AtomicEntryRef()); - // A document cannot be added twice. - assert(!_node_refs[docid].load_acquire().valid()); - - // Note: The level array instance lives as long as the document is present in the index. - LevelArray levels(num_levels, AtomicEntryRef()); - auto node_ref = _nodes.add(levels); - _node_refs[docid].store_release(node_ref); -} - -void -HnswIndex::remove_node_for_document(uint32_t docid) -{ - auto node_ref = _node_refs[docid].load_acquire(); - _nodes.remove(node_ref); - EntryRef invalid; - _node_refs[docid].store_release(invalid); -} - -HnswIndex::LevelArrayRef -HnswIndex::get_level_array(uint32_t docid) const -{ - auto node_ref = _node_refs[docid].load_acquire(); - return _nodes.get(node_ref); -} - -HnswIndex::LinkArrayRef -HnswIndex::get_link_array(uint32_t docid, uint32_t level) const -{ - auto levels = get_level_array(docid); - assert(level < levels.size()); - return _links.get(levels[level].load_acquire()); -} - -void -HnswIndex::set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links) -{ - auto new_links_ref = _links.add(links); - auto node_ref = _node_refs[docid].load_acquire(); - auto levels = _nodes.get_writable(node_ref); - auto old_links_ref = levels[level].load_acquire(); - levels[level].store_release(new_links_ref); - _links.remove(old_links_ref); -} - bool HnswIndex::have_closer_distance(HnswCandidate candidate, const LinkArrayRef& result) const { @@ -182,7 +136,7 @@ HnswIndex::select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_l void HnswIndex::shrink_if_needed(uint32_t docid, uint32_t level) { - auto old_links = get_link_array(docid, level); + auto old_links = _graph.get_link_array(docid, level); uint32_t max_links = max_links_for_level(level); if (old_links.size() > max_links) { HnswCandidateVector neighbors; @@ -191,7 +145,7 @@ HnswIndex::shrink_if_needed(uint32_t docid, uint32_t level) neighbors.emplace_back(neighbor_docid, dist); } auto split = select_neighbors(neighbors, max_links); - set_link_array(docid, level, split.used); + _graph.set_link_array(docid, level, split.used); for (uint32_t removed_docid : split.unused) { remove_link_to(removed_docid, docid, level); } @@ -201,9 +155,9 @@ HnswIndex::shrink_if_needed(uint32_t docid, uint32_t level) void HnswIndex::connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level) { - set_link_array(docid, level, neighbors); + _graph.set_link_array(docid, level, neighbors); for (uint32_t neighbor_docid : neighbors) { - auto old_links = get_link_array(neighbor_docid, level); + auto old_links = _graph.get_link_array(neighbor_docid, level); add_link_to(neighbor_docid, level, old_links, docid); } for (uint32_t neighbor_docid : neighbors) { @@ -215,11 +169,11 @@ void HnswIndex::remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level) { LinkArray new_links; - auto old_links = get_link_array(remove_from, level); + auto old_links = _graph.get_link_array(remove_from, level); for (uint32_t id : old_links) { if (id != remove_id) new_links.push_back(id); } - set_link_array(remove_from, level, new_links); + _graph.set_link_array(remove_from, level, new_links); } @@ -244,7 +198,7 @@ HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& e bool keep_searching = true; while (keep_searching) { keep_searching = false; - for (uint32_t neighbor_docid : get_link_array(nearest.docid, level)) { + for (uint32_t neighbor_docid : _graph.get_link_array(nearest.docid, level)) { double dist = calc_distance(input, neighbor_docid); if (dist < nearest.distance) { nearest = HnswCandidate(neighbor_docid, dist); @@ -259,7 +213,7 @@ void HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level) const { NearestPriQ candidates; - uint32_t doc_id_limit = _node_refs.size(); + uint32_t doc_id_limit = _graph.node_refs.size(); auto visited = _visited_set_pool.get(doc_id_limit); for (const auto &entry : best_neighbors.peek()) { assert(entry.docid < doc_id_limit); @@ -274,7 +228,7 @@ HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, Fur break; } candidates.pop(); - for (uint32_t neighbor_docid : get_link_array(cand.docid, level)) { + for (uint32_t neighbor_docid : _graph.get_link_array(cand.docid, level)) { if ((neighbor_docid >= doc_id_limit) || visited.is_marked(neighbor_docid)) { continue; } @@ -294,15 +248,12 @@ HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, Fur HnswIndex::HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func, RandomLevelGenerator::UP level_generator, const Config& cfg) - : _vectors(vectors), + : + _graph(), + _vectors(vectors), _distance_func(std::move(distance_func)), _level_generator(std::move(level_generator)), - _cfg(cfg), - _node_refs(), - _nodes(make_default_node_store_config()), - _links(make_default_link_store_config()), - _entry_docid(0), // Note that docid 0 is reserved and never used - _entry_level(-1) + _cfg(cfg) { } @@ -314,16 +265,16 @@ HnswIndex::add_document(uint32_t docid) auto input = get_vector(docid); // TODO: Add capping on num_levels int level = _level_generator->max_level(); - make_node_for_document(docid, level + 1); - if (_entry_docid == 0) { - _entry_docid = docid; - _entry_level = level; + _graph.make_node_for_document(docid, level + 1); + uint32_t entry_docid = get_entry_docid(); + if (entry_docid == 0) { + _graph.set_entry_node(docid, level); return; } - int search_level = _entry_level; - double entry_dist = calc_distance(input, _entry_docid); - HnswCandidate entry_point(_entry_docid, entry_dist); + int search_level = get_entry_level(); + double entry_dist = calc_distance(input, entry_docid); + HnswCandidate entry_point(entry_docid, entry_dist); while (search_level > level) { entry_point = find_nearest_in_layer(input, entry_point, search_level); --search_level; @@ -331,7 +282,7 @@ HnswIndex::add_document(uint32_t docid) FurthestPriQ best_neighbors; best_neighbors.push(entry_point); - search_level = std::min(level, _entry_level); + search_level = std::min(level, search_level); // Insert the added document in each level it should exist in. while (search_level >= 0) { @@ -341,9 +292,8 @@ HnswIndex::add_document(uint32_t docid) connect_new_node(docid, neighbors.used, search_level); --search_level; } - if (level > _entry_level) { - _entry_docid = docid; - _entry_level = level; + if (level > get_entry_level()) { + _graph.set_entry_node(docid, level); } } @@ -353,7 +303,7 @@ HnswIndex::mutual_reconnect(const LinkArrayRef &cluster, uint32_t level) std::vector<PairDist> pairs; for (uint32_t i = 0; i + 1 < cluster.size(); ++i) { uint32_t n_id_1 = cluster[i]; - LinkArrayRef n_list_1 = get_link_array(n_id_1, level); + LinkArrayRef n_list_1 = _graph.get_link_array(n_id_1, level); for (uint32_t j = i + 1; j < cluster.size(); ++j) { uint32_t n_id_2 = cluster[j]; if (has_link_to(n_list_1, n_id_2)) continue; @@ -362,10 +312,10 @@ HnswIndex::mutual_reconnect(const LinkArrayRef &cluster, uint32_t level) } std::sort(pairs.begin(), pairs.end()); for (const PairDist & pair : pairs) { - LinkArrayRef old_links_1 = get_link_array(pair.id_first, level); + LinkArrayRef old_links_1 = _graph.get_link_array(pair.id_first, level); if (old_links_1.size() >= _cfg.max_links_on_inserts()) continue; - LinkArrayRef old_links_2 = get_link_array(pair.id_second, level); + LinkArrayRef old_links_2 = _graph.get_link_array(pair.id_second, level); if (old_links_2.size() >= _cfg.max_links_on_inserts()) continue; add_link_to(pair.id_first, level, old_links_1, pair.id_second); @@ -376,27 +326,25 @@ HnswIndex::mutual_reconnect(const LinkArrayRef &cluster, uint32_t level) void HnswIndex::remove_document(uint32_t docid) { - bool need_new_entrypoint = (docid == _entry_docid); + bool need_new_entrypoint = (docid == get_entry_docid()); LinkArray empty; - LevelArrayRef node_levels = get_level_array(docid); + LevelArrayRef node_levels = _graph.get_level_array(docid); for (int level = node_levels.size(); level-- > 0; ) { - LinkArrayRef my_links = get_link_array(docid, level); + LinkArrayRef my_links = _graph.get_link_array(docid, level); for (uint32_t neighbor_id : my_links) { if (need_new_entrypoint) { - _entry_docid = neighbor_id; - _entry_level = level; + _graph.set_entry_node(neighbor_id, level); need_new_entrypoint = false; } remove_link_to(neighbor_id, docid, level); } mutual_reconnect(my_links, level); - set_link_array(docid, level, empty); + _graph.set_link_array(docid, level, empty); } if (need_new_entrypoint) { - _entry_docid = 0; - _entry_level = -1; + _graph.set_entry_node(0, -1); } - remove_node_for_document(docid); + _graph.remove_node_for_document(docid); } void @@ -404,26 +352,26 @@ HnswIndex::transfer_hold_lists(generation_t current_gen) { // Note: RcuVector transfers hold lists as part of reallocation based on current generation. // We need to set the next generation here, as it is incremented on a higher level right after this call. - _node_refs.setGeneration(current_gen + 1); - _nodes.transferHoldLists(current_gen); - _links.transferHoldLists(current_gen); + _graph.node_refs.setGeneration(current_gen + 1); + _graph.nodes.transferHoldLists(current_gen); + _graph.links.transferHoldLists(current_gen); } void HnswIndex::trim_hold_lists(generation_t first_used_gen) { - _node_refs.removeOldGenerations(first_used_gen); - _nodes.trimHoldLists(first_used_gen); - _links.trimHoldLists(first_used_gen); + _graph.node_refs.removeOldGenerations(first_used_gen); + _graph.nodes.trimHoldLists(first_used_gen); + _graph.links.trimHoldLists(first_used_gen); } vespalib::MemoryUsage HnswIndex::memory_usage() const { vespalib::MemoryUsage result; - result.merge(_node_refs.getMemoryUsage()); - result.merge(_nodes.getMemoryUsage()); - result.merge(_links.getMemoryUsage()); + result.merge(_graph.node_refs.getMemoryUsage()); + result.merge(_graph.nodes.getMemoryUsage()); + result.merge(_graph.links.getMemoryUsage()); result.merge(_visited_set_pool.memory_usage()); return result; } @@ -433,6 +381,34 @@ HnswIndex::get_state(const vespalib::slime::Inserter& inserter) const { auto& object = inserter.insertObject(); StateExplorerUtils::memory_usage_to_slime(memory_usage(), object.setObject("memory_usage")); + object.setLong("nodes", _graph.size()); + auto& histogram_array = object.setArray("level_histogram"); + auto level_histogram = _graph.level_histogram(); + for (uint32_t hist_val : level_histogram) { + histogram_array.addLong(hist_val); + } + uint32_t reachable = count_reachable_nodes(); + uint32_t unreachable = _graph.size() - reachable; + if (level_histogram.size() > 0) { + unreachable -= level_histogram[0]; + } + object.setLong("unreachable_nodes", unreachable); + object.setLong("entry_docid", _graph.entry_docid); + object.setLong("entry_level", _graph.entry_level); +} + +std::unique_ptr<NearestNeighborIndexSaver> +HnswIndex::make_saver() const +{ + return std::make_unique<HnswIndexSaver>(_graph); +} + +bool +HnswIndex::load(const fileutil::LoadedBuffer& buf) +{ + assert(get_entry_docid() == 0); // cannot load after index has data + HnswIndexLoader loader(_graph); + return loader.load(buf); } struct NeighborsByDocId { @@ -463,12 +439,13 @@ FurthestPriQ HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k) const { FurthestPriQ best_neighbors; - if (_entry_level < 0) { + if (get_entry_level() < 0) { return best_neighbors; } - double entry_dist = calc_distance(vector, _entry_docid); - HnswCandidate entry_point(_entry_docid, entry_dist); - int search_level = _entry_level; + uint32_t entry_docid = get_entry_docid(); + int search_level = get_entry_level(); + double entry_dist = calc_distance(vector, entry_docid); + HnswCandidate entry_point(entry_docid, entry_dist); while (search_level > 0) { entry_point = find_nearest_in_layer(vector, entry_point, search_level); --search_level; @@ -481,14 +458,14 @@ HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k) const HnswNode HnswIndex::get_node(uint32_t docid) const { - auto node_ref = _node_refs[docid].load_acquire(); + auto node_ref = _graph.node_refs[docid].load_acquire(); if (!node_ref.valid()) { return HnswNode(); } - auto levels = _nodes.get(node_ref); + auto levels = _graph.nodes.get(node_ref); HnswNode::LevelArray result; for (const auto& links_ref : levels) { - auto links = _links.get(links_ref.load_acquire()); + auto links = _graph.links.get(links_ref.load_acquire()); HnswNode::LinkArray result_links(links.begin(), links.end()); std::sort(result_links.begin(), result_links.end()); result.push_back(result_links); @@ -501,14 +478,13 @@ HnswIndex::set_node(uint32_t docid, const HnswNode &node) { size_t num_levels = node.size(); assert(num_levels > 0); - make_node_for_document(docid, num_levels); + _graph.make_node_for_document(docid, num_levels); for (size_t level = 0; level < num_levels; ++level) { connect_new_node(docid, node.level(level), level); } int max_level = num_levels - 1; - if (_entry_level < max_level) { - _entry_docid = docid; - _entry_level = max_level; + if (get_entry_level() < max_level) { + _graph.set_entry_node(docid, max_level); } } @@ -516,15 +492,15 @@ bool HnswIndex::check_link_symmetry() const { bool all_sym = true; - for (size_t docid = 0; docid < _node_refs.size(); ++docid) { - auto node_ref = _node_refs[docid].load_acquire(); + for (size_t docid = 0; docid < _graph.node_refs.size(); ++docid) { + auto node_ref = _graph.node_refs[docid].load_acquire(); if (node_ref.valid()) { - auto levels = _nodes.get(node_ref); + auto levels = _graph.nodes.get(node_ref); uint32_t level = 0; for (const auto& links_ref : levels) { - auto links = _links.get(links_ref.load_acquire()); + auto links = _graph.links.get(links_ref.load_acquire()); for (auto neighbor_docid : links) { - auto neighbor_links = get_link_array(neighbor_docid, level); + auto neighbor_links = _graph.get_link_array(neighbor_docid, level); if (! has_link_to(neighbor_links, docid)) { all_sym = false; } @@ -536,4 +512,31 @@ HnswIndex::check_link_symmetry() const return all_sym; } +uint32_t +HnswIndex::count_reachable_nodes() const +{ + int search_level = get_entry_level(); + if (search_level < 0) { + return 0; + } + auto visited = _visited_set_pool.get(_graph.size()); + uint32_t entry_id = get_entry_docid(); + LinkArray found_links; + found_links.push_back(entry_id); + visited.mark(entry_id); + while (search_level >= 0) { + for (uint32_t idx = 0; idx < found_links.size(); ++idx) { + uint32_t docid = found_links[idx]; + auto neighbors = _graph.get_link_array(docid, search_level); + for (uint32_t neighbor : neighbors) { + if (visited.is_marked(neighbor)) continue; + visited.mark(neighbor); + found_links.push_back(neighbor); + } + } + --search_level; + } + return found_links.size(); +} + } // namespace diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 130c012effe..95001853710 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -8,6 +8,7 @@ #include "hnsw_node.h" #include "nearest_neighbor_index.h" #include "random_level_generator.h" +#include "hnsw_graph.h" #include <vespa/eval/tensor/dense/typed_cells.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/vespalib/datastore/array_store.h> @@ -57,54 +58,30 @@ public: }; protected: - using AtomicEntryRef = search::datastore::AtomicEntryRef; + using AtomicEntryRef = HnswGraph::AtomicEntryRef; + using NodeStore = HnswGraph::NodeStore; - // This uses 10 bits for buffer id -> 1024 buffers. - // As we have very short arrays we get less fragmentation with fewer and larger buffers. - using EntryRefType = search::datastore::EntryRefT<22>; - - // Provides mapping from document id -> node reference. - // The reference is used to lookup the node data in NodeStore. - using NodeRefVector = vespalib::RcuVector<AtomicEntryRef>; + using LinkStore = HnswGraph::LinkStore; + using LinkArrayRef = HnswGraph::LinkArrayRef; + using LinkArray = vespalib::Array<uint32_t>; - // This stores the level arrays for all nodes. - // Each node consists of an array of levels (from level 0 to n) where each entry is a reference to the link array at that level. - using NodeStore = search::datastore::ArrayStore<AtomicEntryRef, EntryRefType>; - using LevelArrayRef = NodeStore::ConstArrayRef; + using LevelArrayRef = HnswGraph::LevelArrayRef; using LevelArray = vespalib::Array<AtomicEntryRef>; - // This stores all link arrays. - // A link array consists of the document ids of the nodes a particular node is linked to. - using LinkStore = search::datastore::ArrayStore<uint32_t, EntryRefType>; - using LinkArrayRef = LinkStore::ConstArrayRef; - using LinkArray = vespalib::Array<uint32_t>; - using TypedCells = vespalib::tensor::TypedCells; + HnswGraph _graph; const DocVectorAccess& _vectors; DistanceFunction::UP _distance_func; RandomLevelGenerator::UP _level_generator; Config _cfg; - NodeRefVector _node_refs; - NodeStore _nodes; - LinkStore _links; mutable vespalib::ReusableSetPool _visited_set_pool; - uint32_t _entry_docid; - int _entry_level; - - static search::datastore::ArrayStoreConfig make_default_node_store_config(); - static search::datastore::ArrayStoreConfig make_default_link_store_config(); uint32_t max_links_for_level(uint32_t level) const; - void make_node_for_document(uint32_t docid, uint32_t num_levels); - void remove_node_for_document(uint32_t docid); - LevelArrayRef get_level_array(uint32_t docid) const; - LinkArrayRef get_link_array(uint32_t docid, uint32_t level) const; - void set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links); void add_link_to(uint32_t docid, uint32_t level, const LinkArrayRef& old_links, uint32_t new_link) { LinkArray new_links(old_links.begin(), old_links.end()); new_links.push_back(new_link); - set_link_array(docid, level, new_links); + _graph.set_link_array(docid, level, new_links); } /** @@ -155,18 +132,25 @@ public: vespalib::MemoryUsage memory_usage() const override; void get_state(const vespalib::slime::Inserter& inserter) const override; + std::unique_ptr<NearestNeighborIndexSaver> make_saver() const override; + bool load(const fileutil::LoadedBuffer& buf) override; + std::vector<Neighbor> find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const override; const DistanceFunction *distance_function() const override { return _distance_func.get(); } FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k) const; - uint32_t get_entry_docid() const { return _entry_docid; } - uint32_t get_entry_level() const { return _entry_level; } + uint32_t get_entry_docid() const { return _graph.entry_docid; } + int32_t get_entry_level() const { return _graph.entry_level; } // Should only be used by unit tests. HnswNode get_node(uint32_t docid) const; void set_node(uint32_t docid, const HnswNode &node); bool check_link_symmetry() const; + uint32_t count_reachable_nodes() const; + + static search::datastore::ArrayStoreConfig make_default_node_store_config(); + static search::datastore::ArrayStoreConfig make_default_link_store_config(); }; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp new file mode 100644 index 00000000000..f02ead86a8d --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp @@ -0,0 +1,47 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "hnsw_index_loader.h" +#include "hnsw_graph.h" +#include <vespa/searchlib/util/fileutil.h> + +namespace search::tensor { + +HnswIndexLoader::~HnswIndexLoader() {} + +HnswIndexLoader::HnswIndexLoader(HnswGraph &graph) + : _graph(graph), _ptr(nullptr), _end(nullptr), _failed(false) +{ +} + +bool +HnswIndexLoader::load(const fileutil::LoadedBuffer& buf) +{ + size_t num_readable = buf.size(sizeof(uint32_t)); + _ptr = static_cast<const uint32_t *>(buf.buffer()); + _end = _ptr + num_readable; + uint32_t entry_docid = next_int(); + int32_t entry_level = next_int(); + uint32_t num_nodes = next_int(); + std::vector<uint32_t> link_array; + for (uint32_t docid = 0; docid < num_nodes; ++docid) { + uint32_t num_levels = next_int(); + if (num_levels > 0) { + _graph.make_node_for_document(docid, num_levels); + for (uint32_t level = 0; level < num_levels; ++level) { + uint32_t num_links = next_int(); + link_array.clear(); + while (num_links-- > 0) { + link_array.push_back(next_int()); + } + _graph.set_link_array(docid, level, link_array); + } + } + } + if (_failed) return false; + _graph.node_refs.ensure_size(num_nodes); + _graph.set_entry_node(entry_docid, entry_level); + return true; +} + + +} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.h new file mode 100644 index 00000000000..174f66b95ec --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.h @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> + +namespace search::fileutil { class LoadedBuffer; } + +namespace search::tensor { + +class HnswGraph; + +/** + * Implements loading of HNSW graph structure from binary format. + **/ +class HnswIndexLoader { +public: + HnswIndexLoader(HnswGraph &graph); + ~HnswIndexLoader(); + bool load(const fileutil::LoadedBuffer& buf); +private: + HnswGraph &_graph; + const uint32_t *_ptr; + const uint32_t *_end; + bool _failed; + uint32_t next_int() { + if (__builtin_expect((_ptr == _end), false)) { + _failed = true; + return 0; + } + return *_ptr++; + } +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp new file mode 100644 index 00000000000..acff30f8cbf --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "hnsw_index_saver.h" +#include "hnsw_graph.h" +#include <vespa/vespalib/util/bufferwriter.h> + +namespace search::tensor { + +HnswIndexSaver::~HnswIndexSaver() {} + +HnswIndexSaver::HnswIndexSaver(const HnswGraph &graph) + : _graph_links(graph.links), _meta_data() +{ + _meta_data.entry_docid = graph.entry_docid; + _meta_data.entry_level = graph.entry_level; + size_t num_nodes = graph.node_refs.size(); + _meta_data.nodes.reserve(num_nodes); + for (size_t i = 0; i < num_nodes; ++i) { + LevelVector node; + auto node_ref = graph.node_refs[i].load_acquire(); + if (node_ref.valid()) { + auto levels = graph.nodes.get(node_ref); + for (const auto& links_ref : levels) { + auto level = links_ref.load_acquire(); + node.push_back(level); + } + } + _meta_data.nodes.emplace_back(std::move(node)); + } +} + +void +HnswIndexSaver::save(BufferWriter& writer) const +{ + writer.write(&_meta_data.entry_docid, sizeof(uint32_t)); + writer.write(&_meta_data.entry_level, sizeof(int32_t)); + uint32_t num_nodes = _meta_data.nodes.size(); + writer.write(&num_nodes, sizeof(uint32_t)); + for (const auto &node : _meta_data.nodes) { + uint32_t num_levels = node.size(); + writer.write(&num_levels, sizeof(uint32_t)); + for (auto links_ref : node) { + if (links_ref.valid()) { + vespalib::ConstArrayRef<uint32_t> link_array = _graph_links.get(links_ref); + uint32_t num_links = link_array.size(); + writer.write(&num_links, sizeof(uint32_t)); + writer.write(link_array.cbegin(), sizeof(uint32_t)*num_links); + } else { + uint32_t num_links = 0; + writer.write(&num_links, sizeof(uint32_t)); + } + } + } + writer.flush(); +} + +} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h new file mode 100644 index 00000000000..d1d8e0db19d --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h @@ -0,0 +1,37 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "nearest_neighbor_index_saver.h" +#include "hnsw_graph.h" +#include <vespa/vespalib/datastore/entryref.h> +#include <vector> + +namespace search::tensor { + +/** + * Implements saving of HNSW graph structure in binary format. + * The constructor takes a snapshot of all meta-data, but + * the links will be fetched from the graph in the save() + * method. + **/ +class HnswIndexSaver : public NearestNeighborIndexSaver { +public: + using LevelVector = std::vector<search::datastore::EntryRef>; + + HnswIndexSaver(const HnswGraph &graph); + ~HnswIndexSaver(); + void save(BufferWriter& writer) const override; + +private: + struct MetaData { + uint32_t entry_docid; + int32_t entry_level; + std::vector<LevelVector> nodes; + MetaData() : entry_docid(0), entry_level(-1), nodes() {} + }; + const HnswGraph::LinkStore &_graph_links; + MetaData _meta_data; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index e7302028996..aca2ce2af66 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -2,17 +2,22 @@ #pragma once -#include <cstdint> -#include <vector> +#include "distance_function.h" #include <vespa/eval/tensor/dense/typed_cells.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/util/memoryusage.h> -#include "distance_function.h" +#include <cstdint> +#include <memory> +#include <vector> namespace vespalib::slime { struct Inserter; } +namespace search::fileutil { class LoadedBuffer; } + namespace search::tensor { +class NearestNeighborIndexSaver; + /** * Interface for an index that is used for (approximate) nearest neighbor search. */ @@ -35,6 +40,15 @@ public: virtual vespalib::MemoryUsage memory_usage() const = 0; virtual void get_state(const vespalib::slime::Inserter& inserter) const = 0; + /** + * Creates a saver that is used to save the index to binary form. + * + * This function is always called by the attribute write thread, + * and the caller ensures that an attribute read guard is held during the lifetime of the saver. + */ + virtual std::unique_ptr<NearestNeighborIndexSaver> make_saver() const = 0; + virtual bool load(const fileutil::LoadedBuffer& buf) = 0; + virtual std::vector<Neighbor> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) const = 0; diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.cpp new file mode 100644 index 00000000000..4b293488737 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.cpp @@ -0,0 +1,3 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "nearest_neighbor_index_saver.h" diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.h new file mode 100644 index 00000000000..99d8960ae10 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.h @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search { class BufferWriter; } + +namespace search::tensor { + +/** + * Interface that is used to save a nearest neighbor index to binary form. + * + * An instance of this interface must hold a snapshot of the index from the + * point in time the instance was created, and then save this to binary form in the save() function. + * + * The instance is always created by the attribute write thread, + * and the caller ensures that an attribute read guard is held during the lifetime of the saver. + * Data that might change later must be copied in the constructor. + * + * A flush thread is calling save() at a later point in time. + */ +class NearestNeighborIndexSaver { +public: + virtual ~NearestNeighborIndexSaver() {} + + /** + * Saves the index in binary form using the given writer. + * + * It is the responsibility of the implementer to call BufferWriter::flush() at the end. + */ + virtual void save(BufferWriter& writer) const = 0; +}; + +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java index 299ea47b257..d113b34f3c6 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java @@ -32,6 +32,7 @@ import java.util.stream.Collectors; * @author freva */ public abstract class InfraApplication implements InfraApplicationApi { + private static final TenantName TENANT_NAME = TenantName.from("hosted-vespa"); private final ApplicationId applicationId; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java index b4b61682a33..4e7a557ffc7 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java @@ -14,7 +14,9 @@ import java.util.Optional; * @author hakonhall */ public interface InfraApplicationApi { + ApplicationId getApplicationId(); Capacity getCapacity(); ClusterSpec getClusterSpecWithVersion(Version version); + } diff --git a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h index 777230566f2..ea5ccf0659b 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h +++ b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h @@ -108,7 +108,7 @@ public: /** * This fetches the object without modifying the lru list. */ - const V & get(const K & key) { return HashTable::find(key)->second._value; } + const V & get(const K & key) const { return HashTable::find(key)->second._value; } /** * This simply erases the object. @@ -133,13 +133,11 @@ public: insert_result insert(const K & key, V && value); /** - * Return the object with the given key. If it does not exist an empty one will be created. - * This can be used as an insert. - * Object is then put at head of LRU list. + * Return pointer to the object with the given key. + * Object is then put at head of LRU list if found. + * If not found nullptr is returned. */ - const V & operator [] (const K & key) const { - return const_cast<lrucache_map<P> *>(this)->findAndRef(key).second._value; - } + V * findAndRef(const K & key); /** * Return the object with the given key. If it does not exist an empty one will be created. @@ -183,7 +181,6 @@ private: * Implements the resize of the hashtable */ void move(NodeStore && oldStore) override; - internal_iterator findAndRef(const K & key); void ref(const internal_iterator & it); insert_result insert(value_type && value); void removeOld(); diff --git a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp index d8d55c9b8c4..839a93cc5ca 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp +++ b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp @@ -263,14 +263,17 @@ lrucache_map<P>::operator [] (const K & key) } template< typename P > -typename lrucache_map<P>::internal_iterator +typename P::Value * lrucache_map<P>::findAndRef(const K & key) { internal_iterator found = HashTable::find(key); if (found != HashTable::end()) { - ref(found); + if (size()*2 > capacity()) { + ref(found); + } + return &found->second._value; } - return found; + return nullptr; } } diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp index 73612452b09..90eb18c23ef 100644 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp @@ -15,14 +15,16 @@ SingleExecutor::SingleExecutor(uint32_t taskLimit) _producerCondition(), _thread(*this), _lastAccepted(0), - _maxPending(0), + _queueSize(), _wakeupConsumerAt(0), _producerNeedWakeupAt(0), - _wp(0) + _wp(0), + _closed(false) { _thread.start(); } SingleExecutor::~SingleExecutor() { + shutdown(); sync(); _thread.stop().join(); } @@ -32,16 +34,6 @@ SingleExecutor::getNumThreads() const { return 1; } -uint64_t -SingleExecutor::addTask(Task::UP task) { - Lock guard(_mutex); - wait_for_room(guard); - uint64_t wp = _wp.load(std::memory_order_relaxed); - _tasks[index(wp)] = std::move(task); - _wp.store(wp + 1, std::memory_order_release); - return wp; -} - void SingleExecutor::sleepProducer(Lock & lock, duration maxWaitTime, uint64_t wakeupAt) { _producerNeedWakeupAt.store(wakeupAt, std::memory_order_relaxed); @@ -51,7 +43,17 @@ SingleExecutor::sleepProducer(Lock & lock, duration maxWaitTime, uint64_t wakeup Executor::Task::UP SingleExecutor::execute(Task::UP task) { - uint64_t wp = addTask(std::move(task)); + uint64_t wp; + { + Lock guard(_mutex); + if (_closed) { + return task; + } + wait_for_room(guard); + wp = _wp.load(std::memory_order_relaxed); + _tasks[index(wp)] = std::move(task); + _wp.store(wp + 1, std::memory_order_release); + } if (wp == _wakeupConsumerAt.load(std::memory_order_relaxed)) { _consumerCondition.notify_one(); } @@ -88,6 +90,13 @@ SingleExecutor::sync() { return *this; } +SingleExecutor & +SingleExecutor::shutdown() { + Lock lock(_mutex); + _closed = true; + return *this; +} + void SingleExecutor::run() { while (!_thread.stopped()) { @@ -112,10 +121,6 @@ SingleExecutor::drain_tasks() { void SingleExecutor::run_tasks_till(uint64_t available) { uint64_t consumed = _rp.load(std::memory_order_relaxed); - uint64_t left = available - consumed; - if (_maxPending.load(std::memory_order_relaxed) < left) { - _maxPending.store(left, std::memory_order_relaxed); - } uint64_t wakeupLimit = _producerNeedWakeupAt.load(std::memory_order_relaxed); while (consumed < available) { Task::UP task = std::move(_tasks[index(consumed)]); @@ -137,6 +142,7 @@ SingleExecutor::wait_for_room(Lock & lock) { _taskLimit = _wantedTaskLimit.load(); taskLimit = _taskLimit; } + _queueSize.add(numTasks()); while (numTasks() >= _taskLimit.load(std::memory_order_relaxed)) { sleepProducer(lock, 10ms, wp - taskLimit/4); } @@ -144,10 +150,11 @@ SingleExecutor::wait_for_room(Lock & lock) { ThreadExecutor::Stats SingleExecutor::getStats() { + Lock lock(_mutex); uint64_t accepted = _wp.load(std::memory_order_relaxed); - Stats stats(_maxPending, (accepted - _lastAccepted), 0); + Stats stats(_queueSize, (accepted - _lastAccepted), 0); _lastAccepted = accepted; - _maxPending = 0; + _queueSize = Stats::QueueSizeT() ; return stats; } diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h index 9c3ebb4caf7..3d759769ea3 100644 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h @@ -27,9 +27,9 @@ public: size_t getNumThreads() const override; uint32_t getTaskLimit() const { return _taskLimit.load(std::memory_order_relaxed); } Stats getStats() override; + SingleExecutor & shutdown() override; private: using Lock = std::unique_lock<std::mutex>; - uint64_t addTask(Task::UP task); void drain(Lock & lock); void run() override; void drain_tasks(); @@ -52,10 +52,11 @@ private: std::condition_variable _producerCondition; vespalib::Thread _thread; uint64_t _lastAccepted; - std::atomic<uint64_t> _maxPending; + Stats::QueueSizeT _queueSize; std::atomic<uint64_t> _wakeupConsumerAt; std::atomic<uint64_t> _producerNeedWakeupAt; std::atomic<uint64_t> _wp; + bool _closed; }; } diff --git a/standalone-container/src/main/sh/standalone-container.sh b/standalone-container/src/main/sh/standalone-container.sh index b49744ebe49..b8025b9629b 100755 --- a/standalone-container/src/main/sh/standalone-container.sh +++ b/standalone-container/src/main/sh/standalone-container.sh @@ -112,13 +112,13 @@ StartCommand() { local service_regex='^[0-9a-zA-Z_-]+$' if ! [[ "$service" =~ $service_regex ]]; then - Fail "Service must match regex '$service_regex'" + Fail "Service must match regex '$service_regex'" fi local pidfile="$VESPA_HOME/var/run/$service.pid" if [ "$force" = false ] && test -r "$pidfile"; then - echo "$service is already running as PID $(< "$pidfile") according to $pidfile" - return + echo "$service is already running as PID $(< "$pidfile") according to $pidfile" + return fi # common setup @@ -199,71 +199,71 @@ Kill() { local -i now if ! now=$(date +%s); then - Fail "Failed to get the current date in seconds since epoch" + Fail "Failed to get the current date in seconds since epoch" fi local -i timeout=$(( now + 300 )) local has_killed=false while true; do - local ps_output="" - if ! ps_output=$(ps -p "$pid" -o user= -o comm=); then - # success - return - fi - - local user comm - read -r user comm <<< "$ps_output" - - if test "$user" != "$expected_user"; then - echo "Warning: Pid collision ($pid): Expected user $expected_user but found $user." - echo "Will assume original process has died." - return - fi - - if test "$comm" != "$expected_comm"; then - echo "Warning: Pid collision ($pid): Expected program $expected_comm but found $comm." - echo "Will assume original process has died." - return - fi - - if ! "$has_killed"; then - if $force; then - if ! kill -KILL "$pid"; then - Fail "Failed to kill $pid" - fi - else - if ! kill "$pid"; then - Fail "Failed to kill $pid" - fi - fi - - has_killed=true - fi - - sleep 1 - - now=$(date +%s) - if (( now >= timeout )); then - Fail "Process $pid still exists after $timeout seconds, giving up" - fi + local ps_output="" + if ! ps_output=$(ps -p "$pid" -o user= -o comm=); then + # success + return + fi + + local user comm + read -r user comm <<< "$ps_output" + + if test "$user" != "$expected_user"; then + echo "Warning: Pid collision ($pid): Expected user $expected_user but found $user." + echo "Will assume original process has died." + return + fi + + if test "$comm" != "$expected_comm"; then + echo "Warning: Pid collision ($pid): Expected program $expected_comm but found $comm." + echo "Will assume original process has died." + return + fi + + if ! "$has_killed"; then + if $force; then + if ! kill -KILL "$pid"; then + Fail "Failed to kill $pid" + fi + else + if ! kill "$pid"; then + Fail "Failed to kill $pid" + fi + fi + + has_killed=true + fi + + sleep 1 + + now=$(date +%s) + if (( now >= timeout )); then + Fail "Process $pid still exists after $timeout seconds, giving up" + fi done } StopCommand() { local user="$1" - local force="$2" - local service="$3" + local service="$2" + local force="$3" local pidfile="$VESPA_HOME/var/run/$service.pid" if ! test -r "$pidfile"; then - echo "$service is not running" - return + echo "$service is not running" + return fi local pid=$(< "$pidfile") if ! [[ "$pid" =~ ^[0-9]+$ ]]; then - Fail "Pid file '$pidfile' does not contain a valid pid: $pid" + Fail "Pid file '$pidfile' does not contain a valid pid: $pid" fi Kill "$force" "$user" java "$pid" @@ -272,7 +272,7 @@ StopCommand() { Main() { if (( $# == 0 )); then - Usage + Usage fi local command="$1" @@ -284,49 +284,49 @@ Main() { local -a jvm_arguments=() while (( $# > 0 )); do - case "$1" in - --help|-h) Usage ;; - --service|-s) - service="$2" - shift 2 - ;; - --user|-u) - user="$2" - shift 2 - ;; - --force|-f) - force=true - shift - ;; - --) - shift - jvm_arguments=("$@") - break - ;; - *) break ;; - esac + case "$1" in + --help|-h) Usage ;; + --service|-s) + service="$2" + shift 2 + ;; + --user|-u) + user="$2" + shift 2 + ;; + --force|-f) + force=true + shift + ;; + --) + shift + jvm_arguments=("$@") + break + ;; + *) break ;; + esac done # Service name will be included in paths and possibly environment variable # names, so be restrictive. local service_regex='^[a-zA-Z0-9_-]+$' if test -z "$service"; then - Fail "SERVICE not specified" + Fail "SERVICE not specified" elif ! [[ "$service" =~ $service_regex ]]; then - Fail "Service must math the regex '$service_regex'" + Fail "Service must math the regex '$service_regex'" fi if ! getent passwd "$user" &> /dev/null; then - Fail "Bad user ($user): not found in passwd" + Fail "Bad user ($user): not found in passwd" elif test "$(id -un)" != "$user"; then - Fail "${0##*/} must be started by $user" + Fail "${0##*/} must be started by $user" fi case "$command" in - help) Usage ;; - start) StartCommand "$service" "$force" "${jvm_arguments[@]}" ;; - stop) StopCommand "$user" "$service" "$force" "$@" ;; - *) Fail "Unknown command '$command'" ;; + help) Usage ;; + start) StartCommand "$service" "$force" "${jvm_arguments[@]}" ;; + stop) StopCommand "$user" "$service" "$force" "$@" ;; + *) Fail "Unknown command '$command'" ;; esac } diff --git a/storage/src/tests/distributor/btree_bucket_database_test.cpp b/storage/src/tests/distributor/btree_bucket_database_test.cpp index 43d74ca2fb5..a2518272a7f 100644 --- a/storage/src/tests/distributor/btree_bucket_database_test.cpp +++ b/storage/src/tests/distributor/btree_bucket_database_test.cpp @@ -9,8 +9,8 @@ using namespace ::testing; namespace storage::distributor { -INSTANTIATE_TEST_CASE_P(BTreeDatabase, BucketDatabaseTest, - ::testing::Values(std::make_shared<BTreeBucketDatabase>())); +VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(BTreeDatabase, BucketDatabaseTest, + ::testing::Values(std::make_shared<BTreeBucketDatabase>())); using document::BucketId; diff --git a/storage/src/tests/distributor/mapbucketdatabasetest.cpp b/storage/src/tests/distributor/mapbucketdatabasetest.cpp index 0ae4a49530e..2c000f6b5db 100644 --- a/storage/src/tests/distributor/mapbucketdatabasetest.cpp +++ b/storage/src/tests/distributor/mapbucketdatabasetest.cpp @@ -5,7 +5,7 @@ namespace storage::distributor { -INSTANTIATE_TEST_CASE_P(MapDatabase, BucketDatabaseTest, - ::testing::Values(std::make_shared<MapBucketDatabase>())); +VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(MapDatabase, BucketDatabaseTest, + ::testing::Values(std::make_shared<MapBucketDatabase>())); } diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index 2a2a840dd4e..8f5b22aa7fa 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -33,6 +33,8 @@ mbus.rpctargetcache.ttl double default = 600 ## Any value below 1 will be 1. mbus.num_threads int default=4 +mbus.optimize_for enum {LATENCY, THROUGHPUT} default = LATENCY + ## Enable to use above thread pool for encoding replies ## False will use network(fnet) thread mbus.dispatch_on_encode bool default=true diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index dd44c96555b..4bcd92293d3 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -36,8 +36,8 @@ PersistenceThread::PersistenceThread(ServiceLayerComponentRegister& compReg, { std::ostringstream threadName; threadName << "Disk " << _env._partition << " thread " << _stripeId; - _component.reset(new ServiceLayerComponent(compReg, threadName.str())); - _bucketOwnershipNotifier.reset(new BucketOwnershipNotifier(*_component, filestorHandler)); + _component = std::make_unique<ServiceLayerComponent>(compReg, threadName.str()); + _bucketOwnershipNotifier = std::make_unique<BucketOwnershipNotifier>(*_component, filestorHandler); framework::MilliSecTime maxProcessingTime(60 * 1000); framework::MilliSecTime waitTime(1000); _thread = _component->startThread(*this, maxProcessingTime, waitTime); @@ -473,11 +473,10 @@ PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd) const document::Bucket &target(i == 0 ? target1 : target2); uint16_t disk(i == 0 ? lock1.disk : lock2.disk); assert(target.getBucketId().getRawId() != 0); - targets.push_back(TargetInfo( - _env.getBucketDatabase(target.getBucketSpace()).get( + targets.emplace_back(_env.getBucketDatabase(target.getBucketSpace()).get( target.getBucketId(), "PersistenceThread::handleSplitBucket - Target", StorBucketDatabase::CREATE_IF_NONEXISTING), - FileStorHandler::RemapInfo(target, disk))); + FileStorHandler::RemapInfo(target, disk)); targets.back().first->setBucketInfo(_env.getBucketInfo(target, disk)); targets.back().first->disk = disk; } @@ -795,7 +794,7 @@ PersistenceThread::handleCommand(api::StorageCommand& msg) { _context = spi::Context(msg.getLoadType(), msg.getPriority(), msg.getTrace().getLevel()); MessageTracker::UP mtracker(handleCommandSplitByType(msg)); - if (mtracker) { + if (mtracker && ! _context.getTrace().getRoot().isEmpty()) { if (mtracker->getReply()) { mtracker->getReply()->getTrace().getRoot().addChild(_context.getTrace().getRoot()); } else { @@ -844,11 +843,11 @@ PersistenceThread::processMessage(api::StorageMessage& msg) LOG(debug, "Handling command: %s", msg.toString().c_str()); LOG(spam, "Message content: %s", msg.toString(true).c_str()); auto tracker(handleCommand(initiatingCommand)); - if (!tracker.get()) { + if (!tracker) { LOG(debug, "Received unsupported command %s", msg.getType().getName().c_str()); } else { tracker->generateReply(initiatingCommand); - if ((tracker->getReply().get() + if ((tracker->getReply() && tracker->getReply()->getResult().failed()) || tracker->getResult().failed()) { diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 978d434847e..fa2b0cda018 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -415,6 +415,7 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> params.setNumThreads(std::max(1, config->mbus.numThreads)); params.setDispatchOnDecode(config->mbus.dispatchOnDecode); params.setDispatchOnEncode(config->mbus.dispatchOnEncode); + params.setTcpNoDelay(config->mbus.optimizeFor == CommunicationManagerConfig::Mbus::OptimizeFor::LATENCY); params.setIdentity(mbus::Identity(_component.getIdentity())); if (config->mbusport != -1) { diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index 2e5eb115844..0f628f59aac 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -20,7 +20,7 @@ #include <iomanip> #include <sstream> -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace ::testing; @@ -105,11 +105,10 @@ std::string version_as_gtest_string(TestParamInfo<vespalib::Version> info) { } -// TODO replace with INSTANTIATE_TEST_SUITE_P on newer gtest versions -INSTANTIATE_TEST_CASE_P(MultiVersionTest, StorageProtocolTest, - Values(vespalib::Version(6, 240, 0), - vespalib::Version(7, 41, 19)), - version_as_gtest_string); +VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(MultiVersionTest, StorageProtocolTest, + Values(vespalib::Version(6, 240, 0), + vespalib::Version(7, 41, 19)), + version_as_gtest_string); namespace { mbus::Message::UP lastCommand; diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java index 82d859b08bd..55b3af93050 100644 --- a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java +++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java @@ -13,10 +13,12 @@ public class ApiAuthenticator implements ai.vespa.hosted.api.ApiAuthenticator { .map(certificateFile -> ControllerHttpClient.withKeyAndCertificate(Properties.apiEndpoint(), Properties.apiKeyFile(), certificateFile)) - .orElseGet(() -> - ControllerHttpClient.withSignatureKey(Properties.apiEndpoint(), - Properties.apiKeyFile(), - Properties.application())); + .or(() -> Properties.apiKey().map(apiKey -> ControllerHttpClient.withSignatureKey(Properties.apiEndpoint(), + apiKey, + Properties.application()))) + .orElseGet(() -> ControllerHttpClient.withSignatureKey(Properties.apiEndpoint(), + Properties.apiKeyFile(), + Properties.application())); } } diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java index 219996ee9aa..94176bbb658 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.vespa.hadoop.mapreduce.util.TupleTools; import com.yahoo.vespa.hadoop.mapreduce.util.VespaConfiguration; -import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.pig.EvalFunc; import org.apache.pig.PigWarning; import org.apache.pig.data.DataBag; @@ -19,6 +18,9 @@ import org.joda.time.DateTime; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.io.UncheckedIOException; import java.math.BigDecimal; import java.math.BigInteger; import java.util.*; @@ -125,14 +127,11 @@ public class VespaDocumentOperation extends EvalFunc<String> { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } - warn("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1); + warnLog("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1); return null; } if (operation == null) { - if (statusReporter != null) { - statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); - } - warn("No valid operation found. Skipping.", PigWarning.UDF_WARNING_1); + warnLog("No valid operation found. Skipping.", PigWarning.UDF_WARNING_2); return null; } @@ -149,7 +148,7 @@ public class VespaDocumentOperation extends EvalFunc<String> { // create json json = create(operation, docId, fields, properties, inputSchema); if (json == null || json.length() == 0) { - warn("No valid document operation could be created.", PigWarning.UDF_WARNING_1); + warnLog("No valid document operation could be created.", PigWarning.UDF_WARNING_3); return null; } @@ -162,8 +161,8 @@ public class VespaDocumentOperation extends EvalFunc<String> { sb.append("Caught exception processing input row: \n"); sb.append(tuple.toString()); sb.append("\nException: "); - sb.append(ExceptionUtils.getStackTrace(e)); - warn(sb.toString(), PigWarning.UDF_WARNING_1); + sb.append(getStackTraceAsString(e)); + warnLog(sb.toString(), PigWarning.UDF_WARNING_4); return null; } if (statusReporter != null) { @@ -644,4 +643,21 @@ public class VespaDocumentOperation extends EvalFunc<String> { } g.writeEndArray(); } + + // copied from vespajlib for reducing dependency and building with JDK 8 + private static String getStackTraceAsString(Throwable throwable) { + try (StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter, true)) { + throwable.printStackTrace(printWriter); + return stringWriter.getBuffer().toString(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + // wrapper to emit logs + private void warnLog(String msg, PigWarning warning) { + warn(msg, warning); + System.err.println(msg); + } } diff --git a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java index 72d0a2ec069..67003273cac 100644 --- a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java +++ b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java @@ -86,7 +86,6 @@ public class VespaDocumentOperationTest { @Test public void requireThatUDFCorrectlyGeneratesRemoveBagAsMapOperation() throws Exception { - DataBag bag = BagFactory.getInstance().newDefaultBag(); Schema innerObjectSchema = new Schema(); @@ -249,6 +248,26 @@ public class VespaDocumentOperationTest { } @Test + public void requireThatUDFReturnsNullWhenExceptionHappens() throws IOException { + Schema schema = new Schema(); + Tuple tuple = TupleFactory.getInstance().newTuple(); + + // broken DELTA format that would throw internally + Map<String, Double> tensor = new HashMap<String, Double>() {{ + put("xlabel1", 2.0); // missing : between 'x' and 'label1' + }}; + + addToTuple("id", DataType.CHARARRAY, "123", schema, tuple); + addToTuple("tensor", DataType.MAP, tensor, schema, tuple); + + VespaDocumentOperation docOp = new VespaDocumentOperation("docid=empty", "create-tensor-fields=tensor"); + docOp.setInputSchema(schema); + String json = docOp.exec(tuple); + + assertNull(json); + } + + @Test public void requireThatUDFCorrectlyGeneratesRemoveOperation() throws Exception { String json = getDocumentOperationJson("operation=remove", "docid=id:<application>:metrics::<name>-<date>"); ObjectMapper m = new ObjectMapper(); diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java index cc714f38290..a798c2ad6df 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java @@ -62,15 +62,16 @@ public class DeployMojo extends AbstractVespaDeploymentMojo { private void tailLogs(ApplicationId id, ZoneId zone, long run) throws MojoFailureException, MojoExecutionException { DeploymentLog log = controller.followDeploymentUntilDone(id, zone, run, this::print); switch (log.status()) { - case success: return; - case error: throw new MojoExecutionException("Unexpected error during deployment; see log for details"); - case aborted: throw new MojoFailureException("Deployment was aborted, probably by a newer deployment"); - case outOfCapacity: throw new MojoFailureException("No capacity left in zone; please contact the Vespa team"); - case deploymentFailed: throw new MojoFailureException("Deployment failed; see log for details"); - case installationFailed: throw new MojoFailureException("Installation failed; see Vespa log for details"); - case running: throw new MojoFailureException("Deployment not completed"); - case testFailure: throw new IllegalStateException("Unexpected status; tests are not run for manual deployments"); - default: throw new IllegalArgumentException("Unexpected status '" + log.status() + "'"); + case success: return; + case error: throw new MojoExecutionException("Unexpected error during deployment; see log for details"); + case aborted: throw new MojoFailureException("Deployment was aborted, probably by a newer deployment"); + case outOfCapacity: throw new MojoFailureException("No capacity left in zone; please contact the Vespa team"); + case deploymentFailed: throw new MojoFailureException("Deployment failed; see log for details"); + case installationFailed: throw new MojoFailureException("Installation failed; see Vespa log for details"); + case running: throw new MojoFailureException("Deployment not completed"); + case endpointCertificateTimeout: throw new MojoFailureException("Endpoint certificate not ready in time; please contact Vespa team"); + case testFailure: throw new IllegalStateException("Unexpected status; tests are not run for manual deployments"); + default: throw new IllegalArgumentException("Unexpected status '" + log.status() + "'"); } } diff --git a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java index 98394a56694..c1e164f7fe8 100644 --- a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java +++ b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java @@ -34,7 +34,8 @@ class FeederParams { private boolean benchmarkMode = false; private int numDispatchThreads = 1; private int maxPending = 0; - private int numConnectionsPerTarget = 2; + private int numConnectionsPerTarget = 1; + private long numMessagesToSend = Long.MAX_VALUE; private List<InputStream> inputStreams = new ArrayList<>(); FeederParams() { @@ -84,10 +85,9 @@ class FeederParams { } int getNumConnectionsPerTarget() { return numConnectionsPerTarget; } - FeederParams setNumConnectionsPerTarget(int numConnectionsPerTarget) { - this.numConnectionsPerTarget = numConnectionsPerTarget; - return this; - } + + long getNumMessagesToSend() { return numMessagesToSend; } + boolean isSerialTransferEnabled() { return maxPending == 1; } @@ -116,6 +116,7 @@ class FeederParams { opts.addOption("b", "mode", true, "Mode for benchmarking."); opts.addOption("o", "output", true, "File to write to. Extensions gives format (.xml, .json, .vespa) json will be produced if no extension."); opts.addOption("c", "numconnections", true, "Number of connections per host."); + opts.addOption("l", "nummessages", true, "Number of messages to send (all is default)."); CommandLine cmd = new DefaultParser().parse(opts, args); @@ -142,6 +143,9 @@ class FeederParams { if (cmd.hasOption('s')) { setSerialTransfer(); } + if (cmd.hasOption('l')) { + numMessagesToSend = Long.valueOf(cmd.getOptionValue('l').trim()); + } if ( !cmd.getArgList().isEmpty()) { inputStreams.clear(); diff --git a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java index 2925ea08de9..556d9bd60c7 100644 --- a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java +++ b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java @@ -65,6 +65,7 @@ public class SimpleFeeder implements ReplyHandler { private final RPCMessageBus mbus; private final SourceSession session; private final int numThreads; + private final long numMessagesToSend; private final Destination destination; private final boolean benchmarkMode; private final static long REPORT_INTERVAL = TimeUnit.SECONDS.toMillis(10); @@ -81,18 +82,20 @@ public class SimpleFeeder implements ReplyHandler { private final Destination destination; private final FeedReader reader; private final Executor executor; - AtomicReference<Throwable> failure; + private final long messagesToSend; + private final AtomicReference<Throwable> failure; - Metrics(Destination destination, FeedReader reader, Executor executor, AtomicReference<Throwable> failure) { + Metrics(Destination destination, FeedReader reader, Executor executor, AtomicReference<Throwable> failure, long messagesToSend) { this.destination = destination; this.reader = reader; this.executor = executor; + this.messagesToSend = messagesToSend; this.failure = failure; } long feed() throws Throwable { long numMessages = 0; - while (failure.get() == null) { + while ((failure.get() == null) && (numMessages < messagesToSend)) { FeedOperation op = reader.read(); if (op.getType() == FeedOperation.Type.INVALID) { break; @@ -341,6 +344,7 @@ public class SimpleFeeder implements ReplyHandler { inputStreams = params.getInputStreams(); out = params.getStdOut(); numThreads = params.getNumDispatchThreads(); + numMessagesToSend = params.getNumMessagesToSend(); mbus = newMessageBus(docTypeMgr, params); session = newSession(mbus, this, params.getMaxPending()); docTypeMgr.configure(params.getConfigId()); @@ -380,7 +384,7 @@ public class SimpleFeeder implements ReplyHandler { printHeader(out); long numMessagesSent = 0; for (InputStream in : inputStreams) { - Metrics m = new Metrics(destination, createFeedReader(in), executor, failure); + Metrics m = new Metrics(destination, createFeedReader(in), executor, failure, numMessagesToSend); numMessagesSent += m.feed(); } while (failure.get() == null && numReplies.get() < numMessagesSent) { diff --git a/vespa_feed_perf/src/main/sh/vespa-feed-perf b/vespa_feed_perf/src/main/sh/vespa-feed-perf index 466cd2ee98c..d6ccf0e4fc5 100755 --- a/vespa_feed_perf/src/main/sh/vespa-feed-perf +++ b/vespa_feed_perf/src/main/sh/vespa-feed-perf @@ -74,4 +74,4 @@ findhost # END environment bootstrap section -exec java -jar $VESPA_HOME/lib/jars/vespa_feed_perf-jar-with-dependencies.jar "$@" +exec java -XX:+UseParallelGC -XX:ParallelGCThreads=4 -jar $VESPA_HOME/lib/jars/vespa_feed_perf-jar-with-dependencies.jar "$@" diff --git a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java index 8682adc0935..13e307d9973 100644 --- a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java +++ b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java @@ -88,12 +88,19 @@ public class FeederParamsTest { } @Test public void requireThatNumConnectionsAreParsed() throws ParseException, FileNotFoundException { - assertEquals(2, new FeederParams().getNumConnectionsPerTarget()); - assertEquals(17, new FeederParams().parseArgs("-c 17").getNumConnectionsPerTarget()); + assertEquals(1, new FeederParams().getNumConnectionsPerTarget()); + assertEquals(16, new FeederParams().parseArgs("-c 16").getNumConnectionsPerTarget()); assertEquals(17, new FeederParams().parseArgs("--numconnections", "17").getNumConnectionsPerTarget()); } @Test + public void requireThatNumMessagesToSendAreParsed() throws ParseException, FileNotFoundException { + assertEquals(Long.MAX_VALUE, new FeederParams().getNumMessagesToSend()); + assertEquals(18, new FeederParams().parseArgs("-l 18").getNumMessagesToSend()); + assertEquals(19, new FeederParams().parseArgs("--nummessages", "19").getNumMessagesToSend()); + } + + @Test public void requireThatDumpStreamAreParsed() throws ParseException, IOException { assertNull(new FeederParams().getDumpStream()); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index d537e873600..e6bc2211bea 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -53,7 +53,6 @@ class ClientFeederV3 { private final Metric metric; private Instant prevOpsPerSecTime = Instant.now(); private double operationsForOpsPerSec = 0d; - private final Object monitor = new Object(); private final StreamReaderV3 streamReaderV3; private final AtomicInteger ongoingRequests = new AtomicInteger(0); diff --git a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java index cceac7e84bb..c455929bf51 100644 --- a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java +++ b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java @@ -16,6 +16,14 @@ import com.yahoo.collections.Pair; */ public class ProcessExecuter { + private final boolean override_log_control; + public ProcessExecuter(boolean override_log_control) { + this.override_log_control = override_log_control; + } + public ProcessExecuter() { + this(false); + } + /** * Executes the given command synchronously without timeout. * @@ -39,6 +47,10 @@ public class ProcessExecuter { ProcessBuilder pb = new ProcessBuilder(command); StringBuilder ret = new StringBuilder(); pb.environment().remove("VESPA_LOG_TARGET"); + if (override_log_control) { + pb.environment().remove("VESPA_LOG_CONTROL_FILE"); + pb.environment().put("VESPA_SERVICE_NAME", "exec-" + command[0]); + } pb.redirectErrorStream(true); Process p = pb.start(); InputStream is = p.getInputStream(); diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index 88a5a05738b..e4631e28625 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -159,7 +159,7 @@ template <> std::vector<double> TestBase<DoubleUniqueStore>::values{ 10.0, 20.0, 30.0, 10.0 }; using UniqueStoreTestTypes = ::testing::Types<NumberUniqueStore, StringUniqueStore, CStringUniqueStore, DoubleUniqueStore>; -TYPED_TEST_CASE(TestBase, UniqueStoreTestTypes); +VESPA_GTEST_TYPED_TEST_SUITE(TestBase, UniqueStoreTestTypes); // Disable warnings emitted by gtest generated files when using typed tests #pragma GCC diagnostic push diff --git a/vespalib/src/tests/executor/executor_test.cpp b/vespalib/src/tests/executor/executor_test.cpp index 9015391beaa..942b425be72 100644 --- a/vespalib/src/tests/executor/executor_test.cpp +++ b/vespalib/src/tests/executor/executor_test.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/vespalib/util/executor_stats.h> using namespace vespalib; @@ -24,4 +25,29 @@ TEST("require that lambdas can be wrapped as tasks") { EXPECT_TRUE(called); } +template<typename T> +void verify(const AggregatedAverage<T> & avg, size_t expCount, T expTotal, T expMin, T expMax, double expAvg) { + EXPECT_EQUAL(expCount, avg.count()); + EXPECT_EQUAL(expTotal, avg.total()); + EXPECT_EQUAL(expMin, avg.min()); + EXPECT_EQUAL(expMax, avg.max()); + EXPECT_EQUAL(expAvg, avg.average()); +} + +TEST("test that aggregated averages") { + TEST_DO(verify(AggregatedAverage<size_t>(), 0ul, 0ul, std::numeric_limits<size_t>::max(), std::numeric_limits<size_t>::min(), 0.0)); + AggregatedAverage<size_t> avg; + avg.add(9); + TEST_DO(verify(avg, 1ul, 9ul, 9ul, 9ul, 9.0)); + avg.add(8); + TEST_DO(verify(avg, 2ul, 17ul, 8ul, 9ul, 8.5)); + avg.add(3, 17, 4,17); + TEST_DO(verify(avg, 5ul, 34ul, 4ul, 17ul, 6.8)); + AggregatedAverage<size_t> avg2; + avg2.add(avg); + TEST_DO(verify(avg2, 5ul, 34ul, 4ul, 17ul, 6.8)); + avg2 += avg; + TEST_DO(verify(avg2, 10ul, 68ul, 4ul, 17ul, 6.8)); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/executor/threadstackexecutor_test.cpp b/vespalib/src/tests/executor/threadstackexecutor_test.cpp index 987b10526a3..9d69adcd96a 100644 --- a/vespalib/src/tests/executor/threadstackexecutor_test.cpp +++ b/vespalib/src/tests/executor/threadstackexecutor_test.cpp @@ -78,10 +78,10 @@ struct MyState { EXPECT_EQUAL(expect_rejected, stats.rejectedTasks); EXPECT_TRUE(!(gate.getCount() == 1) || (expect_deleted == 0)); if (expect_deleted == 0) { - EXPECT_EQUAL(expect_queue + expect_running, stats.maxPendingTasks); + EXPECT_EQUAL(expect_queue + expect_running, stats.queueSize.max()); } stats = executor.getStats(); - EXPECT_EQUAL(expect_queue + expect_running, stats.maxPendingTasks); + EXPECT_EQUAL(expect_queue + expect_running, stats.queueSize.max()); EXPECT_EQUAL(0u, stats.acceptedTasks); EXPECT_EQUAL(0u, stats.rejectedTasks); return *this; @@ -187,12 +187,18 @@ TEST_F("require that executor thread stack tag can be set", ThreadStackExecutor( } TEST("require that stats can be accumulated") { - ThreadStackExecutor::Stats stats(1,2,3); - EXPECT_EQUAL(1u, stats.maxPendingTasks); + ThreadStackExecutor::Stats stats(ThreadExecutor::Stats::QueueSizeT(1) ,2,3); + EXPECT_EQUAL(1u, stats.queueSize.max()); EXPECT_EQUAL(2u, stats.acceptedTasks); EXPECT_EQUAL(3u, stats.rejectedTasks); - stats += ThreadStackExecutor::Stats(7,8,9); - EXPECT_EQUAL(8u, stats.maxPendingTasks); + stats += ThreadStackExecutor::Stats(ThreadExecutor::Stats::QueueSizeT(7),8,9); + EXPECT_EQUAL(2u, stats.queueSize.count()); + EXPECT_EQUAL(8u, stats.queueSize.total()); + EXPECT_EQUAL(8u, stats.queueSize.max()); + EXPECT_EQUAL(8u, stats.queueSize.min()); + EXPECT_EQUAL(8u, stats.queueSize.max()); + EXPECT_EQUAL(4.0, stats.queueSize.average()); + EXPECT_EQUAL(10u, stats.acceptedTasks); EXPECT_EQUAL(12u, stats.rejectedTasks); diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp index b39b6859623..d23c2c6b68c 100644 --- a/vespalib/src/tests/stllike/hash_test.cpp +++ b/vespalib/src/tests/stllike/hash_test.cpp @@ -32,7 +32,7 @@ namespace { TEST("test that hashValue gives expected response") { const char * s("abcdefghi"); - EXPECT_EQUAL(7045194595191919248ul, vespalib::hashValue(s)); + EXPECT_EQUAL(2878261200250560019ul, vespalib::hashValue(s)); EXPECT_EQUAL(vespalib::hashValue(s), vespalib::hashValue(s, strlen(s))); EXPECT_NOT_EQUAL(vespalib::hashValue(s), vespalib::hashValue(s, strlen(s)-1)); } diff --git a/vespalib/src/tests/stllike/lookup_benchmark.cpp b/vespalib/src/tests/stllike/lookup_benchmark.cpp index acde9ea8f9a..b3ce8c29a18 100644 --- a/vespalib/src/tests/stllike/lookup_benchmark.cpp +++ b/vespalib/src/tests/stllike/lookup_benchmark.cpp @@ -5,7 +5,6 @@ #include <set> #include <unordered_set> #include <vector> -//#define XXH_INLINE_ALL #include <xxhash.h> #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> diff --git a/vespalib/src/vespa/vespalib/gtest/gtest.h b/vespalib/src/vespa/vespalib/gtest/gtest.h index e5bfcf2ae55..87362687103 100644 --- a/vespalib/src/vespa/vespalib/gtest/gtest.h +++ b/vespalib/src/vespa/vespalib/gtest/gtest.h @@ -14,3 +14,15 @@ main(int argc, char* argv[]) \ ::testing::InitGoogleTest(&argc, argv); \ return RUN_ALL_TESTS(); \ } + +#ifdef INSTANTIATE_TEST_SUITE_P +#define VESPA_GTEST_INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_SUITE_P +#else +#define VESPA_GTEST_INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_CASE_P +#endif + +#ifdef TYPED_TEST_SUITE +#define VESPA_GTEST_TYPED_TEST_SUITE TYPED_TEST_SUITE +#else +#define VESPA_GTEST_TYPED_TEST_SUITE TYPED_TEST_CASE +#endif diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp index d8c6c87ecda..5f4fee06c4a 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp @@ -1,25 +1,20 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "hash_fun.h" +#include <xxhash.h> namespace vespalib { size_t hashValue(const char *str) { - size_t res = 0; - unsigned const char *pt = (unsigned const char *) str; - while (*pt != 0) { - res = (res << 7) + (res >> 25) + *pt++; - } - return res; + return hashValue(str, strlen(str)); } /** * @brief Calculate hash value. * - * This is the hash function used by the HashMap class. - * The hash function is inherited from Fastserver4 / FastLib / pandora. + * The hash function XXH64 from xxhash library. * @param buf input buffer * @param sz input buffer size * @return hash value of input @@ -27,12 +22,7 @@ hashValue(const char *str) size_t hashValue(const void * buf, size_t sz) { - size_t res = 0; - unsigned const char *pt = (unsigned const char *) buf; - for (size_t i(0); i < sz; i++) { - res = (res << 7) + (res >> 25) + pt[i]; - } - return res; + return XXH64(buf, sz, 0); } } diff --git a/vespalib/src/vespa/vespalib/util/executor_stats.h b/vespalib/src/vespa/vespalib/util/executor_stats.h index 771435fbabf..9b941095c27 100644 --- a/vespalib/src/vespa/vespalib/util/executor_stats.h +++ b/vespalib/src/vespa/vespalib/util/executor_stats.h @@ -2,21 +2,69 @@ #pragma once +#include <limits> + namespace vespalib { /** + * Used for aggregating values, preserving min, max, sum and count. + */ +template <typename T> +class AggregatedAverage { +public: + AggregatedAverage() : AggregatedAverage(0ul, T(0), std::numeric_limits<T>::max(), std::numeric_limits<T>::min()) { } + explicit AggregatedAverage(T value) : AggregatedAverage(1, value, value, value) { } + AggregatedAverage(size_t count_in, T total_in, T min_in, T max_in) + : _count(count_in), + _total(total_in), + _min(min_in), + _max(max_in) + { } + AggregatedAverage & operator += (const AggregatedAverage & rhs) { + add(rhs); + return *this; + } + void add(const AggregatedAverage & rhs) { + add(rhs._count, rhs._total, rhs._min, rhs._max); + } + void add(T value) { + add(1, value, value, value); + } + void add(size_t count_in, T total_in, T min_in, T max_in) { + _count += count_in; + _total += total_in; + if (min_in < _min) _min = min_in; + if (max_in > _max) _max = max_in; + } + size_t count() const { return _count; } + T total() const { return _total; } + T min() const { return _min; } + T max() const { return _max; } + double average() const { return (_count > 0) ? (double(_total) / _count) : 0; } +private: + size_t _count; + T _total; + T _min; + T _max; +}; + +/** * Struct representing stats for an executor. **/ struct ExecutorStats { - size_t maxPendingTasks; + using QueueSizeT = AggregatedAverage<size_t>; + QueueSizeT queueSize; size_t acceptedTasks; size_t rejectedTasks; - ExecutorStats() : ExecutorStats(0, 0, 0) {} - ExecutorStats(size_t maxPending, size_t accepted, size_t rejected) - : maxPendingTasks(maxPending), acceptedTasks(accepted), rejectedTasks(rejected) + ExecutorStats() : ExecutorStats(QueueSizeT(), 0, 0) {} + ExecutorStats(QueueSizeT queueSize_in, size_t accepted, size_t rejected) + : queueSize(queueSize_in), acceptedTasks(accepted), rejectedTasks(rejected) {} ExecutorStats & operator += (const ExecutorStats & rhs) { - maxPendingTasks += rhs.maxPendingTasks; + queueSize = QueueSizeT(queueSize.count() + rhs.queueSize.count(), + queueSize.total() + rhs.queueSize.total(), + queueSize.min() + rhs.queueSize.min(), + queueSize.max() + rhs.queueSize.max()); acceptedTasks += rhs.acceptedTasks; rejectedTasks += rhs.rejectedTasks; return *this; diff --git a/vespalib/src/vespa/vespalib/util/signalhandler.cpp b/vespalib/src/vespa/vespalib/util/signalhandler.cpp index 21543ef10d8..0a5eea0d327 100644 --- a/vespalib/src/vespa/vespalib/util/signalhandler.cpp +++ b/vespalib/src/vespa/vespalib/util/signalhandler.cpp @@ -19,9 +19,6 @@ public: } -// Clear SignalHandler::_handlers in a slightly less unsafe manner. -Shutdown shutdown; - SignalHandler SignalHandler::HUP(SIGHUP); SignalHandler SignalHandler::INT(SIGINT); SignalHandler SignalHandler::TERM(SIGTERM); @@ -36,6 +33,9 @@ SignalHandler SignalHandler::FPE(SIGFPE); SignalHandler SignalHandler::QUIT(SIGQUIT); SignalHandler SignalHandler::USR1(SIGUSR1); +// Clear SignalHandler::_handlers in a slightly less unsafe manner. +Shutdown shutdown; + void SignalHandler::handleSignal(int signal) { @@ -112,8 +112,14 @@ SignalHandler::shutdown() it = _handlers.begin(), ite = _handlers.end(); it != ite; ++it) { - if (*it != nullptr) - (*it)->unhook(); + if (*it != nullptr) { + // Ignore SIGTERM at shutdown in case valgrind is used. + if ((*it)->_signal == SIGTERM) { + (*it)->ignore(); + } else { + (*it)->unhook(); + } + } } std::vector<SignalHandler *>().swap(_handlers); } diff --git a/vespalib/src/vespa/vespalib/util/threadexecutor.h b/vespalib/src/vespa/vespalib/util/threadexecutor.h index 202e516bc60..61a5d9d5ac7 100644 --- a/vespalib/src/vespa/vespalib/util/threadexecutor.h +++ b/vespalib/src/vespa/vespalib/util/threadexecutor.h @@ -40,6 +40,7 @@ public: class SyncableThreadExecutor : public ThreadExecutor, public Syncable { public: + virtual SyncableThreadExecutor & shutdown() = 0; }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp index 9ab465841d6..efb1dbf4054 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp @@ -196,7 +196,7 @@ ThreadStackExecutorBase::getStats() LockGuard lock(_monitor); Stats stats = _stats; _stats = Stats(); - _stats.maxPendingTasks = _taskCount; + _stats.queueSize.add(_taskCount); return stats; } @@ -208,8 +208,7 @@ ThreadStackExecutorBase::execute(Task::UP task) TaggedTask taggedTask(std::move(task), _barrier.startEvent()); ++_taskCount; ++_stats.acceptedTasks; - _stats.maxPendingTasks = (_taskCount > _stats.maxPendingTasks) - ?_taskCount : _stats.maxPendingTasks; + _stats.queueSize.add(_taskCount); if (!_workers.empty()) { Worker *worker = _workers.back(); _workers.popBack(); diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h index 2c0bc56d6df..6333a8fc66e 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h @@ -226,7 +226,7 @@ public: * * @return this object; for chaining **/ - ThreadStackExecutorBase &shutdown(); + ThreadStackExecutorBase &shutdown() override; /** * Will invoke shutdown then sync. diff --git a/vespalog/abi-spec.json b/vespalog/abi-spec.json index 09ac3fa75d3..996cc0259a0 100644 --- a/vespalog/abi-spec.json +++ b/vespalog/abi-spec.json @@ -186,19 +186,6 @@ ], "fields": [] }, - "com.yahoo.log.MappedLevelControllerRepo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.nio.MappedByteBuffer, int, int, java.lang.String)", - "public com.yahoo.log.LevelController getLevelController(java.lang.String)", - "public void checkBack()" - ], - "fields": [] - }, "com.yahoo.log.RejectFilter": { "superClass": "java.lang.Object", "interfaces": [], @@ -300,15 +287,10 @@ "public" ], "methods": [ - "public void <init>(java.lang.String, java.lang.String, java.lang.String)", - "public com.yahoo.log.LevelController getLevelControl(java.lang.String)", "public com.yahoo.log.LevelController getLevelController(java.lang.String)", "public void close()" ], - "fields": [ - "public static final int controlFileHeaderLength", - "public static final int numLevels" - ] + "fields": [] }, "com.yahoo.log.event.Collection": { "superClass": "com.yahoo.log.event.Event", diff --git a/vespalog/src/main/java/com/yahoo/log/LevelController.java b/vespalog/src/main/java/com/yahoo/log/LevelController.java index ccd18f126d6..0efe0d4e7c1 100644 --- a/vespalog/src/main/java/com/yahoo/log/LevelController.java +++ b/vespalog/src/main/java/com/yahoo/log/LevelController.java @@ -1,4 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.util.logging.Level; + /** * This is the interface for controlling the log level of a * component logger. This hides the actual controlling @@ -7,32 +11,24 @@ * @author arnej27959 * */ - -/** - * @author arnej27959 - **/ -package com.yahoo.log; - -import java.util.logging.Level; - public interface LevelController { /** * should we actually publish a log message with the given Level now? - **/ - public boolean shouldLog(Level level); + */ + boolean shouldLog(Level level); /** * return a string suitable for printing in a logctl file. * the string must be be 4 * 8 characters, where each group * of 4 characters is either " ON" or " OFF". - **/ - public String getOnOffString(); + */ + String getOnOffString(); /** * check the current state of logging and reflect it into the * associated Logger instance, if available. - **/ - public void checkBack(); - public Level getLevelLimit(); + */ + void checkBack(); + Level getLevelLimit(); } diff --git a/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java index f02d8793b23..53f4de4f264 100644 --- a/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java +++ b/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java @@ -13,14 +13,14 @@ import java.util.Map; * @author Ulf Lilleengen * @since 5.1 */ -public class MappedLevelControllerRepo { +class MappedLevelControllerRepo { private final Map<String, LevelController> levelControllerMap = new HashMap<>(); private final MappedByteBuffer mapBuf; private final int controlFileHeaderLength; private final int numLevels; private final String logControlFilename; - public MappedLevelControllerRepo(MappedByteBuffer mapBuf, int controlFileHeaderLength, int numLevels, String logControlFilename) { + MappedLevelControllerRepo(MappedByteBuffer mapBuf, int controlFileHeaderLength, int numLevels, String logControlFilename) { this.mapBuf = mapBuf; this.controlFileHeaderLength = controlFileHeaderLength; this.numLevels = numLevels; @@ -101,12 +101,12 @@ public class MappedLevelControllerRepo { return MappedLevelController.checkOnOff(mapBuf, levstart); } - public LevelController getLevelController(String suffix) { + LevelController getLevelController(String suffix) { return levelControllerMap.get(suffix); } - public void checkBack() { + void checkBack() { for (LevelController ctrl : levelControllerMap.values()) { ctrl.checkBack(); } diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java index 85d92075827..2cc88855deb 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java @@ -30,12 +30,12 @@ public class VespaLevelControllerRepo implements LevelControllerRepo { /** * length of fixed header content of a control file, constant: **/ - public static final int controlFileHeaderLength; + static final int controlFileHeaderLength; /** * number of distinctly controlled levels (in logctl files), * must be compatible with C++ Vespa logging **/ - public static final int numLevels = 8; + static final int numLevels = 8; static { controlFileHeaderLength = CFHEADER.length() @@ -50,7 +50,7 @@ public class VespaLevelControllerRepo implements LevelControllerRepo { **/ private LevelController defaultLevelCtrl; - public VespaLevelControllerRepo(String logCtlFn, String logLevel, String applicationPrefix) { + VespaLevelControllerRepo(String logCtlFn, String logLevel, String applicationPrefix) { this.logControlFilename = logCtlFn; this.appPrefix = applicationPrefix; defaultLevelCtrl = new DefaultLevelController(logLevel); @@ -142,7 +142,7 @@ public class VespaLevelControllerRepo implements LevelControllerRepo { levelControllerRepo = new MappedLevelControllerRepo(mapBuf, controlFileHeaderLength, numLevels, logControlFilename); } - public LevelController getLevelControl(String suffix) { + private LevelController getLevelControl(String suffix) { LevelController ctrl = null; if (levelControllerRepo != null) { if (suffix == null || suffix.equals("default")) { diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java index 331780f226b..32b1003c20c 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java @@ -44,7 +44,7 @@ class VespaLogHandler extends StreamHandler { /** * Publish a log record into the Vespa log target. */ - public synchronized void publish (LogRecord record) { + public synchronized void publish(LogRecord record) { Level level = record.getLevel(); String component = record.getLoggerName(); diff --git a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java index c0dd856b634..220e5e9271e 100644 --- a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java @@ -13,6 +13,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.util.LinkedList; import java.util.List; import java.util.concurrent.BrokenBarrierException; @@ -32,20 +33,20 @@ import static org.junit.Assert.fail; * @author Bjorn Borud */ public class VespaLogHandlerTestCase { - protected static String hostname; - protected static String pid; + private static String hostname; + private static String pid; - protected static LogRecord record1; - protected static String record1String; + static LogRecord record1; + static String record1String; - protected static LogRecord record2; - protected static String record2String; + static LogRecord record2; + private static String record2String; - protected static LogRecord record3; - protected static String record3String; + private static LogRecord record3; + private static String record3String; - protected static LogRecord record4; - protected static String record4String; + private static LogRecord record4; + private static String record4String; static { hostname = Util.getHostName(); @@ -139,7 +140,7 @@ public class VespaLogHandlerTestCase { } @Test - public void testFallback() throws FileNotFoundException { + public void testFallback() { File file = new File("mydir2"); file.delete(); assertTrue(file.mkdir()); @@ -157,7 +158,7 @@ public class VespaLogHandlerTestCase { * Perform simple test */ @Test - public void testLogCtl () throws InterruptedException, FileNotFoundException { + public void testLogCtl () { MockLevelController ctl = new MockLevelController(); MockLevelControllerRepo ctlRepo = new MockLevelControllerRepo(ctl); MockLogTarget target = new MockLogTarget(); @@ -203,7 +204,7 @@ public class VespaLogHandlerTestCase { @Test public void testRotate () throws IOException { // Doesn't work in Windows. TODO: Fix the logging stuff - if (System.getProperty("os.name").toLowerCase().indexOf("win")>=0) + if (System.getProperty("os.name").toLowerCase().contains("win")) return; try { VespaLogHandler h @@ -269,10 +270,8 @@ public class VespaLogHandlerTestCase { ); class LogRacer implements Runnable { - private int n; - public LogRacer (int n) { - this.n = n; + private LogRacer() { } public void run () { @@ -285,7 +284,7 @@ public class VespaLogHandlerTestCase { } } - public void logLikeCrazy () { + void logLikeCrazy() { for (int j = 0; j < numLogEntries; j++) { try { h.publish(record1); @@ -299,7 +298,7 @@ public class VespaLogHandlerTestCase { } for (int i = 0; i < numThreads; i++) { - t[i] = new Thread(new LogRacer(i)); + t[i] = new Thread(new LogRacer()); t[i].start(); } @@ -361,35 +360,23 @@ public class VespaLogHandlerTestCase { * */ protected static String[] readFile (String fileName) { - BufferedReader br = null; - List<String> lines = new LinkedList<String>(); - try { - br = new BufferedReader( - new InputStreamReader(new FileInputStream(new File(fileName)), "UTF-8")); + List<String> lines = new LinkedList<>(); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(new File(fileName)), StandardCharsets.UTF_8))) { for (String line = br.readLine(); line != null; - line = br.readLine()) - { + line = br.readLine()) { lines.add(line); } - return lines.toArray(new String[lines.size()]); - } - catch (Throwable e) { + return lines.toArray(new String[0]); + } catch (Throwable e) { return new String[0]; } - finally { - if (br != null) { - try { - br.close(); - } - catch (IOException e) {} - } - } } private static class MockLevelControllerRepo implements LevelControllerRepo { private LevelController levelController; - public MockLevelControllerRepo(LevelController controller) { + MockLevelControllerRepo(LevelController controller) { this.levelController = controller; } @@ -411,7 +398,7 @@ public class VespaLogHandlerTestCase { return (level.equals(logLevel)); } - public void setShouldLog(Level level) { + void setShouldLog(Level level) { this.logLevel = level; } @@ -431,7 +418,7 @@ public class VespaLogHandlerTestCase { private static class MockLogTarget implements LogTarget { private final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - public String[] getLines() { + String[] getLines() { return baos.toString().split("\n"); } @Override diff --git a/vespalog/src/vespa/log/llparser.cpp b/vespalog/src/vespa/log/llparser.cpp index 8e44f36c7ae..ae1af3e6416 100644 --- a/vespalog/src/vespa/log/llparser.cpp +++ b/vespalog/src/vespa/log/llparser.cpp @@ -361,8 +361,7 @@ LLParser::makeMessage(const char *tmf, const char *hsf, const char *pdf, if ((c == '\\' && src[0] == 't') || (c >= 32 && c < '\\') || (c > '\\' && c < 128) - || c == 0 - || c > 160) + || c == 0) { *dst++ = static_cast<char>(c); } else { diff --git a/vsm/src/tests/textutil/textutil.cpp b/vsm/src/tests/textutil/textutil.cpp index e71b95f22f9..581419fc80e 100644 --- a/vsm/src/tests/textutil/textutil.cpp +++ b/vsm/src/tests/textutil/textutil.cpp @@ -60,10 +60,10 @@ void TextUtilTest::assertSkipSeparators(const char * input, size_t len, const UCS4V & expdstbuf, const SizeV & expoffsets) { const byte * srcbuf = reinterpret_cast<const byte *>(input); - ucs4_t dstbuf[len]; - size_t offsets[len]; + auto dstbuf = std::make_unique<ucs4_t[]>(len + 1); + auto offsets = std::make_unique<size_t[]>(len + 1); UTF8StrChrFieldSearcher fs; - BW bw(dstbuf, offsets); + BW bw(dstbuf.get(), offsets.get()); size_t dstlen = fs.skipSeparators(srcbuf, len, bw); EXPECT_EQUAL(dstlen, expdstbuf.size()); ASSERT_TRUE(dstlen == expdstbuf.size()); |