diff options
155 files changed, 2401 insertions, 1886 deletions
diff --git a/bootstrap-cmake.sh b/bootstrap-cmake.sh index bc127da7a0b..13e49767964 100644 --- a/bootstrap-cmake.sh +++ b/bootstrap-cmake.sh @@ -25,6 +25,6 @@ cmake3 \ -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ -DEXTRA_LINK_DIRECTORY="/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm3.9/lib" \ -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-boost/include;/opt/vespa-cppunit/include;/usr/include/llvm3.9" \ - -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/include/llvm3.9" \ + -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/lib64/llvm3.9/lib" \ ${EXTRA_CMAKE_ARGS} \ "${SOURCE_DIR}" diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java index 3fec8550623..b8b329f2b04 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java @@ -53,6 +53,8 @@ public interface Model { * once per deployment. * @param fileDistribution {@link com.yahoo.config.model.api.FileDistribution} instance. */ + // TODO: Remove when 6.206 is the oldest version in use + @Deprecated default void reloadDeployFileDistributor(FileDistribution fileDistribution) { } /** diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index a69835626ea..7b28edbb2fc 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -46,7 +46,7 @@ public interface ModelContext { boolean hostedVespa(); Zone zone(); Set<Rotation> rotations(); - default boolean disableFileDistributor() { return false; } + default boolean disableFileDistributor() { return true; } // TODO: Remove when oldest version in use is 6.206 } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index eb0c6067fca..d6b916680d8 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -68,7 +68,6 @@ public class DeployState implements ConfigDefinitionStore { private final Version wantedNodeVespaVersion; private final Instant now; private final HostProvisioner provisioner; - private final boolean disableFiledistributor; public static DeployState createTestState() { return new Builder().build(); @@ -82,7 +81,7 @@ public class DeployState implements ConfigDefinitionStore { FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, DeployProperties properties, Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo, java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, - SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion, boolean disableFiledistributor) { + SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.rankProfileRegistry = rankProfileRegistry; @@ -101,7 +100,6 @@ public class DeployState implements ConfigDefinitionStore { this.validationOverrides = applicationPackage.getValidationOverrides().map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); this.wantedNodeVespaVersion = wantedNodeVespaVersion; this.now = now; - this.disableFiledistributor = disableFiledistributor; } public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { @@ -215,8 +213,6 @@ public class DeployState implements ConfigDefinitionStore { public Instant now() { return now; } - public boolean disableFiledistributor() { return disableFiledistributor; } - public static class Builder { private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty(); @@ -231,7 +227,6 @@ public class DeployState implements ConfigDefinitionStore { private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; - private boolean disableFiledistributor = false; public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; @@ -293,11 +288,6 @@ public class DeployState implements ConfigDefinitionStore { return this; } - public Builder disableFiledistributor(boolean disableFiledistributor) { - this.disableFiledistributor = disableFiledistributor; - return this; - } - public DeployState build() { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage); @@ -305,7 +295,7 @@ public class DeployState implements ConfigDefinitionStore { SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles); return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner, properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, - zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion, disableFiledistributor); + zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion); } private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index 1de7ce62df7..2bfc7a418de 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -29,9 +29,7 @@ public class ExactMatch extends Processor { public void process() { for (SDField field : search.allConcreteFields()) { Matching.Type matching = field.getMatching().getType(); - if (matching.equals(Matching.Type.EXACT) || - matching.equals(Matching.Type.WORD)) - { + if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { implementExactMatch(field, search); } else if (field.getMatching().getExactMatchTerminator() != null) { warn(search, field, "exact-terminator requires 'exact' matching to have any effect."); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java index a4d7b1b4054..02655906f65 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java @@ -25,7 +25,7 @@ import java.util.Set; import java.util.TreeSet; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class TextMatch extends Processor { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 47a644a39d4..a4130c8052c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -412,11 +412,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri } @Override - public void reloadDeployFileDistributor(FileDistribution fileDistribution) { - getFileDistributor().reloadDeployFileDistributor(fileDistribution); - } - - @Override public AllocatedHosts allocatedHosts() { return allocatedHosts; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 7a1fab8dbd0..74512e70ebe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -138,8 +138,7 @@ public class VespaModelFactory implements ModelFactory { .rotations(modelContext.properties().rotations()) .zone(zone) .now(clock.instant()) - .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()) - .disableFiledistributor(modelContext.properties().disableFileDistributor()); + .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()); modelContext.previousModel().ifPresent(builder::previousModel); return builder.build(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index bce78017bdd..62828b314d0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -144,9 +144,6 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.loadBalancerAddress().isPresent()) { builder.loadBalancerAddress(options.loadBalancerAddress().get()); } - if (options.disableFiledistributor().isPresent()) { - builder.disableFiledistributor(options.disableFiledistributor().get()); - } } private String[] getConfigModelPluginDirs() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index 866bae6666a..0ebd3987ba4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -44,5 +44,4 @@ public interface CloudConfigOptions { Optional<String> dockerRegistry(); Optional<String> dockerVespaBaseImage(); Optional<String> loadBalancerAddress(); - Optional<Boolean> disableFiledistributor(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 81fc464327e..ac49ec53cbf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -85,7 +85,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { /** * Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE */ - static final String HOSTED_VESPA_STATUS_FILE_YINST_SETTING = "cloudconfig_server__tenant_vip_status_file"; + static final String HOSTED_VESPA_STATUS_FILE_INSTALL_SETTING = "cloudconfig_server__tenant_vip_status_file"; public enum Networking { disable, enable } @@ -242,7 +242,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { protected void addStatusHandlers(ContainerCluster cluster, ConfigModelContext configModelContext) { if (configModelContext.getDeployState().isHosted()) { String name = "status.html"; - Optional<String> statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_YINST_SETTING)); + Optional<String> statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_INSTALL_SETTING)); cluster.addComponent( new FileStatusHandlerComponent(name + "-status-handler", statusFile.orElse(HOSTED_VESPA_STATUS_FILE), "http://*/" + name, "https://*/" + name)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java index ffba56fa17f..34e242400d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java @@ -24,9 +24,7 @@ public class FileDistributionConfigProvider { } public void getConfig(FiledistributorrpcConfig.Builder builder) { - // If disabled config proxy should act as file distributor, so use config proxy port - int port = ConfigProxy.BASEPORT; - builder.connectionspec("tcp/" + host.getHostname() + ":" + port); + builder.connectionspec("tcp/" + host.getHostname() + ":" + ConfigProxy.BASEPORT); } public void getConfig(FilereferencesConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index ad27d86fb84..abd4b604a6d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -133,10 +133,4 @@ public class FileDistributor { dbHandler.removeDeploymentsThatHaveDifferentApplicationId(getTargetHostnames()); } - // should only be called during deploy, and only once, since it leads to file distributor - // rescanning all files, which is very expensive ATM (April 2016) - public void reloadDeployFileDistributor(FileDistribution dbHandler) { - dbHandler.reloadDeployFileDistributor(); - } - } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index 01e1a0ba9fa..ad35eff6467 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -306,7 +306,6 @@ public class AdminTestCase { @Test public void testDisableFileDistributorForAllApps() { DeployState state = new DeployState.Builder() - .disableFiledistributor(true) .properties( new DeployProperties.Builder(). zone(new Zone(Environment.dev, RegionName.from("baz"))). diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index 1784fe0e974..c698e9f5079 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -19,7 +19,6 @@ public class TestOptions implements CloudConfigOptions { private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); private Optional<Integer> numParallelTenantLoaders = Optional.empty(); - private Optional<Boolean> disableFiledistributor = Optional.empty(); @Override public Optional<Integer> rpcPort() { @@ -118,9 +117,6 @@ public class TestOptions implements CloudConfigOptions { @Override public Optional<String> loadBalancerAddress() { return Optional.empty(); } - @Override - public Optional<Boolean> disableFiledistributor() { return disableFiledistributor; } - public TestOptions numParallelTenantLoaders(int numLoaders) { this.numParallelTenantLoaders = Optional.of(numLoaders); return this; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index d3879f1ab36..96137652e22 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -265,7 +265,7 @@ public class VespaModelTestCase { assertThat(admin.getConfigservers().size(), is(1)); Set<HostInfo> hosts = model.getHosts(); assertThat(hosts.size(), is(1)); - //logd, config proxy, sentinel, config server, slobrok, log server, file distributor + //logd, config proxy, sentinel, config server, slobrok, log server HostInfo host = hosts.iterator().next(); assertThat(host.getServices().size(), is(6)); new LogdConfig((LogdConfig.Builder) model.getConfig(new LogdConfig.Builder(), "admin/model")); diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index aa807c76c91..494dae8b086 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -4,7 +4,8 @@ namespace=cloud.config # Ports rpcport int default=19070 httpport int default=19071 -numthreads int default=16 +# 0 means use the number of CPU cores available +numRpcThreads int default=0 # ZooKeeper zookeeperserver[].hostname string diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 967cb06a13a..55c08297b43 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -5,6 +5,7 @@ import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.Vtag; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; @@ -384,7 +385,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } void redeployAllApplications(Deployer deployer) throws InterruptedException { - ExecutorService deploymentExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders()); + ExecutorService deploymentExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(), + new DaemonThreadFactory("redeploy apps")); tenants.getAllTenants().forEach(tenant -> listApplicationIds(tenant) .forEach(applicationId -> redeployApplication(applicationId, deployer, deploymentExecutor))); deploymentExecutor.shutdown(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 0435c8e59db..82231fbf5d8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -45,6 +45,12 @@ public interface TenantApplications { Transaction deleteApplication(ApplicationId applicationId); /** + * Removes unused applications + * + */ + void removeUnusedApplications(); + + /** * Closes the application repo. Once a repo has been closed, it should not be used again. */ void close(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java index c3d13b86591..d6f34650f8f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java @@ -18,15 +18,11 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; -import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.logging.Logger; /** @@ -40,13 +36,11 @@ import java.util.logging.Logger; public class ZKTenantApplications implements TenantApplications, PathChildrenCacheListener { private static final Logger log = Logger.getLogger(ZKTenantApplications.class.getName()); - private static final Duration checkForRemovedApplicationsInterval = Duration.ofMinutes(1); private final Curator curator; private final Path applicationsPath; private final ExecutorService pathChildrenExecutor = Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(ZKTenantApplications.class.getName())); - private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1); private final Curator.DirectoryCache directoryCache; private final ReloadHandler reloadHandler; private final TenantName tenant; @@ -60,10 +54,6 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, pathChildrenExecutor); this.directoryCache.start(); this.directoryCache.addListener(this); - checkForRemovedApplicationsService.scheduleWithFixedDelay(this::removeApplications, - checkForRemovedApplicationsInterval.getSeconds(), - checkForRemovedApplicationsInterval.getSeconds(), - TimeUnit.SECONDS); } public static TenantApplications create(Curator curator, ReloadHandler reloadHandler, TenantName tenant) { @@ -130,7 +120,6 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac public void close() { directoryCache.close(); pathChildrenExecutor.shutdown(); - checkForRemovedApplicationsService.shutdown(); } @Override @@ -151,7 +140,7 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac } // We might have lost events and might need to remove applications (new applications are // not added by listening for events here, they are added when session is added, see RemoteSessionRepo) - removeApplications(); + removeUnusedApplications(); } private void applicationRemoved(ApplicationId applicationId) { @@ -163,7 +152,7 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac log.log(LogLevel.DEBUG, Tenants.logPre(applicationId) + "Application added: " + applicationId); } - private void removeApplications() { + public void removeUnusedApplications() { ImmutableSet<ApplicationId> activeApplications = ImmutableSet.copyOf(listApplications()); log.log(LogLevel.DEBUG, "Removing stale applications for tenant '" + tenant + "', not removing these active applications: " + activeApplications); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 4502cc7e223..2f05c1a2259 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -141,7 +141,6 @@ public class ModelContextImpl implements ModelContext { private final boolean hostedVespa; private final Zone zone; private final Set<Rotation> rotations; - private final boolean disableFileDistributor; public Properties(ApplicationId applicationId, boolean multitenant, @@ -149,8 +148,7 @@ public class ModelContextImpl implements ModelContext { HostName loadBalancerName, boolean hostedVespa, Zone zone, - Set<Rotation> rotations, - boolean disableFileDistributor) { + Set<Rotation> rotations) { this.applicationId = applicationId; this.multitenant = multitenant; this.configServerSpecs = configServerSpecs; @@ -158,7 +156,6 @@ public class ModelContextImpl implements ModelContext { this.hostedVespa = hostedVespa; this.zone = zone; this.rotations = rotations; - this.disableFileDistributor = disableFileDistributor; } @Override @@ -194,8 +191,6 @@ public class ModelContextImpl implements ModelContext { @Override public Set<Rotation> rotations() { return rotations; } - @Override - public boolean disableFileDistributor() { return disableFileDistributor; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java index b7567769afd..c39f85ec87f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java @@ -57,7 +57,7 @@ public class FileDistributionImpl implements FileDistribution { log.log(LogLevel.DEBUG, "Executing " + request.methodName() + " against " + target.toString()); target.invokeSync(request, timeout); if (request.isError() && request.errorCode() != ErrorCode.CONNECTION) { - log.log(LogLevel.INFO, request.methodName() + " failed: " + request.errorCode() + " (" + request.errorMessage() + ")"); + log.log(LogLevel.DEBUG, request.methodName() + " failed: " + request.errorCode() + " (" + request.errorMessage() + ")"); } target.close(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLock.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLock.java deleted file mode 100644 index 547c4a1bc2a..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLock.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.filedistribution; - -import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.recipes.CuratorLock; -import com.yahoo.vespa.curator.recipes.CuratorLockException; - -import java.time.Clock; -import java.time.Duration; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -/** - * Global filedistribution lock to ensure only one configserver may work on filedistribution. - * The implementation uses a combination of a {@link java.util.concurrent.locks.ReentrantLock} and - * a {@link CuratorLock} to ensure both mutual exclusion within the JVM and - * across JVMs via ZooKeeper. - * - * @author lulf - */ -public class FileDistributionLock implements Lock { - private final Lock processLock; - private final CuratorLock curatorLock; - - public FileDistributionLock(Curator curator, String zkPath) { - this.processLock = new ReentrantLock(); - this.curatorLock = new CuratorLock(curator, zkPath); - } - - @Override - public void lock() { - processLock.lock(); - try { - curatorLock.lock(); - } catch (CuratorLockException e) { - processLock.unlock(); - throw e; - } - } - - @Override - public void lockInterruptibly() throws InterruptedException { - throw new UnsupportedOperationException(); - - } - - @Override - public boolean tryLock() { - if (processLock.tryLock()) { - if (curatorLock.tryLock()) { - return true; - } else { - processLock.unlock(); - return false; - } - } else { - return false; - } - } - - @Override - public boolean tryLock(long timeout, TimeUnit unit) throws InterruptedException { - TimeoutBudget budget = new TimeoutBudget(Clock.systemUTC(), Duration.ofMillis(unit.toMillis(timeout))); - if (processLock.tryLock(budget.timeLeft().toMillis(), TimeUnit.MILLISECONDS)) { - if (curatorLock.tryLock(budget.timeLeft().toMillis(), TimeUnit.MILLISECONDS)) { - return true; - } else { - processLock.unlock(); - return false; - } - } else { - return false; - } - } - - @Override - public void unlock() { - try { - curatorLock.unlock(); - } finally { - processLock.unlock(); - } - } - - @Override - public Condition newCondition() { - throw new UnsupportedOperationException(); - } -} - diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java index dcc590180f2..d518867407a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java @@ -12,7 +12,6 @@ import java.util.Set; */ public class MockFileDBHandler implements FileDistribution { public int sendDeployedFilesCalled = 0; - public int reloadDeployFileDistributorCalled = 0; public int removeDeploymentsThatHaveDifferentApplicationIdCalled = 0; @Override @@ -26,9 +25,7 @@ public class MockFileDBHandler implements FileDistribution { } @Override - public void reloadDeployFileDistributor() { - reloadDeployFileDistributorCalled++; - } + public void reloadDeployFileDistributor() {} @Override public void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 731e343532a..daff32198a3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -186,8 +186,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { HostName.from(configserverConfig.loadBalancerAddress()), configserverConfig.hostedVespa(), zone, - rotations, - configserverConfig.disableFiledistributor()); + rotations); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 08095c55373..a5f288bf254 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -116,7 +116,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { this.metrics = metrics.getOrCreateMetricUpdater(Collections.<String, String>emptyMap()); this.hostLivenessTracker = hostLivenessTracker; BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(config.maxgetconfigclients()); - executorService = new ThreadPoolExecutor(config.numthreads(), config.numthreads(), + int numberOfRpcThreads = (config.numRpcThreads() == 0) ? Runtime.getRuntime().availableProcessors() : config.numRpcThreads(); + executorService = new ThreadPoolExecutor(numberOfRpcThreads, numberOfRpcThreads, 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getThreadFactory(THREADPOOL_NAME)); delayedConfigResponses = new DelayedConfigResponses(this, config.numDelayedResponseThreads()); spec = new Spec(null, config.rpcport()); @@ -461,7 +462,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { request.parameters().add(new StringValue(fileData.filename())); request.parameters().add(new StringValue(fileData.type().name())); request.parameters().add(new Int64Value(fileData.size())); - target.invokeSync(request, 600); + invokeRpcIfValidConnection(request); if (request.isError()) { log.warning("Failed delivering meta for reference '" + fileData.fileReference().value() + "' with file '" + fileData.filename() + "' to " + target.toString() + " with error: '" + request.errorMessage() + "'."); @@ -479,7 +480,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { request.parameters().add(new Int32Value(session)); request.parameters().add(new Int32Value(partId)); request.parameters().add(new DataValue(buf)); - target.invokeSync(request, 600); + invokeRpcIfValidConnection(request); if (request.isError()) { throw new IllegalArgumentException("Failed delivering reference '" + ref.value() + "' to " + target.toString() + " with error: '" + request.errorMessage() + "'."); @@ -496,7 +497,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { request.parameters().add(new Int64Value(fileData.xxhash())); request.parameters().add(new Int32Value(status.getCode())); request.parameters().add(new StringValue(status.getDescription())); - target.invokeSync(request, 600); + invokeRpcIfValidConnection(request); if (request.isError()) { throw new IllegalArgumentException("Failed delivering reference '" + fileData.fileReference().value() + "' with file '" + fileData.filename() + "' to " + target.toString() + " with error: '" + request.errorMessage() + "'."); @@ -506,6 +507,14 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } } } + + private void invokeRpcIfValidConnection(Request request) { + if (target.isValid()) { + target.invokeSync(request, 600); + } else { + throw new RuntimeException("Connection to " + target + " is invalid", target.getConnectionLostReason()); + } + } } @SuppressWarnings("UnusedDeclaration") diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index fd645c86d1b..bfed526a130 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -102,7 +102,6 @@ public class SessionPreparer { preparation.writeStateZK(); preparation.writeRotZK(); preparation.distribute(); - preparation.reloadDeployFileDistributor(); } log.log(LogLevel.DEBUG, () -> "time used " + params.getTimeoutBudget().timesUsed() + " : " + params.getApplicationId()); @@ -155,8 +154,7 @@ public class SessionPreparer { HostName.from(configserverConfig.loadBalancerAddress()), configserverConfig.hostedVespa(), zone, - rotationsSet, - configserverConfig.disableFiledistributor()); + rotationsSet); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, @@ -221,13 +219,6 @@ public class SessionPreparer { checkTimeout("distribute files"); } - void reloadDeployFileDistributor() { - if (prepareResult.asList().isEmpty()) return; - PreparedModelsBuilder.PreparedModelResult aModelResult = prepareResult.asList().get(0); - aModelResult.model.reloadDeployFileDistributor(aModelResult.fileDistributionProvider.getFileDistribution()); - checkTimeout("reload all deployed files in file distributor"); - } - ConfigChangeActions result() { return prepareResult.getConfigChangeActions(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index 61145c2a138..934ba9b754d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -90,9 +90,8 @@ public class TenantBuilder { * Create a real tenant from the properties given by this builder. * * @return a new {@link Tenant} instance. - * @throws Exception if building fails */ - public Tenant build() throws Exception { + public Tenant build() { createTenantRequestHandler(); createApplicationRepo(); createRemoteSessionFactory(componentRegistry.getClock()); @@ -140,7 +139,7 @@ public class TenantBuilder { private void createSessionCounter() { if (sessionCounter == null) { - sessionCounter = new SessionCounter(componentRegistry.getCurator(), tenant); + sessionCounter = new SessionCounter(componentRegistry.getConfigCurator(), tenant); } } @@ -169,7 +168,7 @@ public class TenantBuilder { } } - private void createRemoteSessionRepo() throws Exception { + private void createRemoteSessionRepo() { if (remoteSessionRepo == null) { remoteSessionRepo = new RemoteSessionRepo(componentRegistry.getCurator(), remoteSessionFactory, @@ -177,6 +176,8 @@ public class TenantBuilder { tenant, applicationRepo, componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenant)), + // TODO: Check if we can avoid using one executor service per tenant. Either one for all + // or have a few executors and hash on tenant name to find out which one to use here createSingleThreadedExecutorService(RemoteSessionRepo.class.getName())); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java index 7e26e7fe8f5..4595c340b64 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java @@ -19,6 +19,7 @@ import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.zookeeper.KeeperException; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -29,6 +30,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -43,7 +46,7 @@ import java.util.logging.Logger; * To create or delete a tenant, the handler calls {@link Tenants#addTenant} and {@link Tenants#deleteTenant} methods. * This will delete shared state from zookeeper, and return, so it does not mean a tenant is immediately deleted. * - * Once a tenant is deleted from zookeeper, the zookeeper watcher thread will get notified on all configservers, and + * Once a tenant is deleted from zookeeper, the zookeeper watcher thread will get notified on all config servers, and * shutdown and delete any per-configserver state. * * @author Vegard Havdal @@ -57,7 +60,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen private static final Path tenantsPath = Path.fromString("/config/v2/tenants/"); private static final Path vespaPath = Path.fromString("/vespa"); - + private static final Duration checkForRemovedApplicationsInterval = Duration.ofMinutes(1); private static final Logger log = Logger.getLogger(Tenants.class.getName()); private final Map<TenantName, Tenant> tenants = new LinkedHashMap<>(); @@ -67,6 +70,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen private final MetricUpdater metricUpdater; private final ExecutorService pathChildrenExecutor = Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(Tenants.class.getName())); + private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1); private final Curator.DirectoryCache directoryCache; @@ -90,10 +94,14 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen this.directoryCache = curator.createDirectoryCache(tenantsPath.getAbsolute(), false, false, pathChildrenExecutor); directoryCache.start(); directoryCache.addListener(this); - log.log(LogLevel.INFO, "Creating all tenants"); // TODO: Change to debug + log.log(LogLevel.DEBUG, "Creating all tenants"); createTenants(); notifyTenantsLoaded(); - log.log(LogLevel.INFO, "All tenants created"); // TODO: Change to debug + log.log(LogLevel.DEBUG, "All tenants created"); + checkForRemovedApplicationsService.scheduleWithFixedDelay(this::removeUnusedApplications, + checkForRemovedApplicationsInterval.getSeconds(), + checkForRemovedApplicationsInterval.getSeconds(), + TimeUnit.SECONDS); } /** @@ -199,6 +207,11 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen return tenants.get(DEFAULT_TENANT); } + + private void removeUnusedApplications() { + getAllTenants().forEach(tenant -> tenant.getApplicationRepo().removeUnusedApplications()); + } + private void notifyNewTenant(Tenant tenant) { for (TenantListener listener : tenantListeners) { listener.onTenantCreate(tenant.getName(), tenant); @@ -328,6 +341,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen public void close() { directoryCache.close(); pathChildrenExecutor.shutdown(); + checkForRemovedApplicationsService.shutdown(); } public boolean checkThatTenantExists(TenantName tenant) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java index 8901bb5b115..dc93886e0c0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.zookeeper; import com.yahoo.log.LogLevel; import com.yahoo.vespa.curator.recipes.CuratorCounter; -import com.yahoo.vespa.curator.Curator; import java.util.ArrayList; import java.util.Collections; @@ -13,8 +12,7 @@ import java.util.List; * A counter that sets its initial value to the number of apps in zookeeper if no counter value is set. Subclass * this to get that behavior. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class InitializedCounter { @@ -22,10 +20,10 @@ public class InitializedCounter { protected final CuratorCounter counter; private final String sessionsDirPath; - public InitializedCounter(Curator curator, String counterPath, String sessionsDirPath) { + public InitializedCounter(ConfigCurator configCurator, String counterPath, String sessionsDirPath) { this.sessionsDirPath = sessionsDirPath; - this.counter = new CuratorCounter(curator, counterPath); - initializeCounterValue(getLatestSessionId(ConfigCurator.create(curator), sessionsDirPath)); + this.counter = new CuratorCounter(configCurator.curator(), counterPath); + initializeCounterValue(getLatestSessionId(configCurator, sessionsDirPath)); } private void initializeCounterValue(Long latestSessionId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java index 4df292dd204..b8f5acdb225 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java @@ -3,18 +3,16 @@ package com.yahoo.vespa.config.server.zookeeper; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.tenant.Tenants; -import com.yahoo.vespa.curator.Curator; /** * A counter keeping track of session ids in an atomic fashion across multiple config servers. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class SessionCounter extends InitializedCounter { - public SessionCounter(Curator curator, TenantName tenantName) { - super(curator, + public SessionCounter(ConfigCurator configCurator, TenantName tenantName) { + super(configCurator, Tenants.getTenantPath(tenantName).append("sessionCounter").getAbsolute(), Tenants.getSessionsPath(tenantName).getAbsolute()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index e4c336a55f1..aed0a6a9750 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -48,8 +48,7 @@ public class ModelContextImplTest { null, false, Zone.defaultZone(), - rotations, - false), + rotations), Optional.empty(), new Version(6), new Version(6)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MemoryTenantApplications.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MemoryTenantApplications.java index 03bcb4d71e9..28b24f15a60 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MemoryTenantApplications.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MemoryTenantApplications.java @@ -51,6 +51,11 @@ public class MemoryTenantApplications implements TenantApplications { } @Override + public void removeUnusedApplications() { + // do nothing + } + + @Override public void close() { isOpen = false; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLockTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLockTest.java deleted file mode 100644 index 362934b7898..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLockTest.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.filedistribution; - -import com.yahoo.vespa.config.server.TestWithCurator; -import com.yahoo.vespa.curator.recipes.CuratorLockException; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.*; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - -/** - * @author lulf - */ -public class FileDistributionLockTest extends TestWithCurator { - - FileDistributionLock lock; - private int value = 0; - - @Before - public void setupLock() { - lock = new FileDistributionLock(curator, "/lock"); - value = 0; - } - - @Test - public void testDistributedLock() throws InterruptedException, TimeoutException, ExecutionException { - ExecutorService executor = Executors.newFixedThreadPool(20); - - List<Future<?>> futureList = new ArrayList<>(); - for (int i = 0; i < 20; i++) { - futureList.add(executor.submit(() -> { - lock.lock(); - value++; - lock.unlock(); - })); - } - - for (Future<?> future : futureList) { - future.get(600, TimeUnit.SECONDS); - } - assertThat(value, is(20)); - } - - @Test - public void testDistributedTryLockFailure() throws InterruptedException { - MockCurator mockCurator = new MockCurator(); - lock = new FileDistributionLock(mockCurator, "/mocklock"); - mockCurator.timeoutOnLock = true; - assertFalse(lock.tryLock(600, TimeUnit.SECONDS)); - mockCurator.timeoutOnLock = false; - // Second time should not be blocking - Thread t = new Thread(() -> { - try { - if (lock.tryLock(6, TimeUnit.SECONDS)) { - value = 1; - lock.unlock(); - } - } catch (InterruptedException e) { - } - }); - assertThat(value, is(0)); - t.start(); - t.join(); - assertThat(value, is(1)); - } - - @Test - public void testDistributedLockExceptionFailure() throws InterruptedException { - MockCurator mockCurator = new MockCurator(); - lock = new FileDistributionLock(mockCurator, "/mocklock"); - mockCurator.throwExceptionOnLock = true; - try { - lock.lock(); - fail("Lock call should not succeed"); - } catch (CuratorLockException e) { - // ignore - } - mockCurator.throwExceptionOnLock = false; - // Second time should not be blocking - Thread t = new Thread(() -> { - try { - lock.lock(); - value = 1; - lock.unlock(); - } catch (Exception e) { - fail("Should not fail"); - } - }); - assertThat(value, is(0)); - t.start(); - t.join(); - assertThat(value, is(1)); - } - - @Test(expected = UnsupportedOperationException.class) - public void testConditionNotSupported() { - lock.newCondition(); - } - - @Test(expected = UnsupportedOperationException.class) - public void testLockInterruptiblyNotSupported() throws InterruptedException { - lock.lockInterruptibly(); - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java index 12dc584f055..e022b622fb0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java @@ -82,7 +82,11 @@ public class TestWithRpc { protected void createAndStartRpcServer(boolean hostedVespa) { ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); - rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder().rpcport(port).numthreads(1).maxgetconfigclients(1).hostedVespa(hostedVespa)), + rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder() + .rpcport(port) + .numRpcThreads(1) + .maxgetconfigclients(1) + .hostedVespa(hostedVespa)), new SuperModelRequestHandler(new TestConfigDefinitionRepo(), configserverConfig, new SuperModelManager( diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java index ac68307f42d..2d182f03de7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java @@ -50,7 +50,7 @@ public class LocalSessionRepoTest extends TestWithCurator { } clock = new ManualClock(Instant.ofEpochSecond(1)); LocalSessionLoader loader = new SessionFactoryImpl(globalComponentRegistry, - new SessionCounter(globalComponentRegistry.getCurator(), tenantName), + new SessionCounter(globalComponentRegistry.getConfigCurator(), tenantName), new MemoryTenantApplications(), tenantFileSystemDirs, new HostRegistry<>(), tenantName); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java index 878339bd703..50c741c494c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java @@ -173,6 +173,11 @@ public class RemoteSessionRepoTest extends TestWithCurator { } @Override + public void removeUnusedApplications() { + // do nothing + } + + @Override public void close() { } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 2069ae48d76..ac16f1f71a7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -126,15 +126,12 @@ public class SessionPreparerTest extends TestWithCurator { new PrepareParams.Builder().dryRun(true).timeoutBudget(TimeoutBudgetTest.day()).build(), Optional.empty(), tenantPath, Instant.now()); assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().sendDeployedFilesCalled, is(0)); - assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().reloadDeployFileDistributorCalled, is(0)); } @Test public void require_that_application_is_prepared() throws Exception { preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), new PrepareParams.Builder().build(), Optional.empty(), tenantPath, Instant.now()); assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().sendDeployedFilesCalled, is(2)); - // Should be called only once no matter how many model versions are built - assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().reloadDeployFileDistributorCalled, is(1)); assertTrue(configCurator.exists(sessionsPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java index 67932726df1..b444e09f558 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java @@ -11,8 +11,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class InitializedCounterTest extends TestWithCurator { @@ -28,7 +27,7 @@ public class InitializedCounterTest extends TestWithCurator { @Test public void requireThatCounterIsInitializedFromNumberOfSessions() { - InitializedCounter counter = new InitializedCounter(curator, "/counter", "/sessions"); + InitializedCounter counter = new InitializedCounter(configCurator, "/counter", "/sessions"); assertThat(counter.counter.get(), is(2l)); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java index a8e5db4f952..e8bc16ca271 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java @@ -26,6 +26,8 @@ public interface ZmsClient { boolean hasTenantAdminAccess(AthenzIdentity athenzIdentity, AthenzDomain tenantDomain); + boolean hasHostedOperatorAccess(AthenzIdentity identity); + // Used before vespa tenancy is established for the domain. boolean isDomainAdmin(AthenzIdentity athenzIdentity, AthenzDomain domain); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 9ee83bec26a..c9f2ff6eb49 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -2,21 +2,20 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; -import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.PersistenceException; @@ -110,9 +109,8 @@ public class TenantController { if (existingTenantWithDomain.isPresent()) throw new IllegalArgumentException("Could not create " + tenant + ": The Athens domain '" + domain.getName() + "' is already connected to " + existingTenantWithDomain.get()); - ZmsClient zmsClient = athenzClientFactory.createZmsClientWithAuthorizedServiceToken(token.get()); - try { zmsClient.deleteTenant(domain); } catch (ZmsException ignored) { } - zmsClient.createTenant(domain); + athenzClientFactory.createZmsClientWithAuthorizedServiceToken(token.get()) + .createTenant(domain); } db.createTenant(tenant); log.info("Created " + tenant); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java index b72cec562d8..81692e790a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java @@ -16,7 +16,7 @@ import java.security.cert.X509Certificate; import java.util.Optional; import java.util.concurrent.Executor; -import static com.yahoo.vespa.hosted.controller.athenz.filter.SecurityFilterUtils.sendErrorResponse; +import static com.yahoo.vespa.hosted.controller.restapi.filter.SecurityFilterUtils.sendErrorResponse; /** * Authenticates Athenz principal, either through: diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java index 80e14ca7f83..1dc46ed81ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java @@ -21,7 +21,7 @@ import java.util.concurrent.Executor; import java.util.logging.Logger; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.controller.athenz.filter.SecurityFilterUtils.sendErrorResponse; +import static com.yahoo.vespa.hosted.controller.restapi.filter.SecurityFilterUtils.sendErrorResponse; /** * A variant of the {@link AthenzPrincipalFilter} to be used in combination with a cookie-based diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java index 8b62a93f8d9..f77e16f67ce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java @@ -102,6 +102,11 @@ public class ZmsClientImpl implements ZmsClient { return hasAccess(TenantAction._modify_.name(), tenantResourceString(tenantDomain), identity); } + @Override + public boolean hasHostedOperatorAccess(AthenzIdentity identity) { + return getOrThrow(() -> hasAccess("modify", service.getDomain() + ":hosted-vespa", identity)); + } + /** * Used when creating tenancies. As there are no tenancy policies at this point, * we cannot use {@link #hasTenantAdminAccess(AthenzIdentity, AthenzDomain)} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java index 0524cf18568..0a360184da9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java @@ -1,13 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.mock; -import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -17,12 +19,18 @@ import java.util.Set; public class AthenzDbMock { public final Map<AthenzDomain, Domain> domains = new HashMap<>(); + public final List<AthenzIdentity> hostedOperators = new ArrayList<>(); public AthenzDbMock addDomain(Domain domain) { domains.put(domain.name, domain); return this; } + public AthenzDbMock addHostedOperator(AthenzIdentity athenzIdentity) { + hostedOperators.add(athenzIdentity); + return this; + } + public static class Domain { public final AthenzDomain name; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java index ba8bfc2405e..3ee2655108a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java @@ -68,17 +68,23 @@ public class ZmsClientMock implements ZmsClient { if (application == null) { throw zmsException(400, "Application '%s' not found", applicationName); } - return domain.admins.contains(identity) || application.acl.get(action).contains(identity); + return isHostedOperator(identity) || domain.admins.contains(identity) || application.acl.get(action).contains(identity); } @Override public boolean hasTenantAdminAccess(AthenzIdentity identity, AthenzDomain tenantDomain) { log("hasTenantAdminAccess(principal='%s', tenantDomain='%s')", identity, tenantDomain); - return isDomainAdmin(identity, tenantDomain) || + return isHostedOperator(identity) || isDomainAdmin(identity, tenantDomain) || getDomainOrThrow(tenantDomain, true).tenantAdmins.contains(identity); } @Override + public boolean hasHostedOperatorAccess(AthenzIdentity identity) { + log("hasHostedOperatorAccess(identity='%s')", identity); + return isHostedOperator(identity); + } + + @Override public boolean isDomainAdmin(AthenzIdentity identity, AthenzDomain domain) { log("isDomainAdmin(principal='%s', domain='%s')", identity, domain); return getDomainOrThrow(domain, false).admins.contains(identity); @@ -109,6 +115,10 @@ public class ZmsClientMock implements ZmsClient { return domain; } + private boolean isHostedOperator(AthenzIdentity identity) { + return athenz.hostedOperators.contains(identity); + } + private static ZmsException zmsException(int code, String message, Object... args) { return new ZmsException(code, String.format(message, args)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java index e9db4f9b717..c6781657b8a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java @@ -98,6 +98,10 @@ public class Path { */ public String getRest() { return rest; } + public String asString() { + return pathString; + } + @Override public String toString() { return "path '" + Arrays.stream(elements).collect(Collectors.joining("/")) + "'"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationInstanceAuthorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationInstanceAuthorizer.java index 283d700c2bd..2deef474f7c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationInstanceAuthorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationInstanceAuthorizer.java @@ -5,6 +5,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantType; @@ -45,6 +46,7 @@ public class ApplicationInstanceAuthorizer { Tenant tenant, ApplicationName application) { AthenzDomain principalDomain = principal.getDomain(); + if (isHostedOperator(principal.getIdentity())) return; if (!principalDomain.equals(SCREWDRIVER_DOMAIN)) { throw loggedForbiddenException( @@ -112,6 +114,11 @@ public class ApplicationInstanceAuthorizer { return new NotAuthorizedException(formattedMessage); } + private boolean isHostedOperator(AthenzIdentity identity) { + return athenzClientFactory.createZmsClientWithServicePrincipal() + .hasHostedOperatorAccess(identity); + } + private boolean hasDeployAccessToAthenzApplication(AthenzPrincipal principal, AthenzDomain domain, ApplicationName application) { try { return athenzClientFactory.createZmsClientWithServicePrincipal() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java index 9d45b9a6e09..6a268ce8fda 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java @@ -95,7 +95,7 @@ public class Authorizer { return new ForbiddenException(formattedMessage); } - private boolean isTenantAdmin(AthenzIdentity identity, Tenant tenant) { + public boolean isTenantAdmin(AthenzIdentity identity, Tenant tenant) { switch (tenant.tenantType()) { case ATHENS: return isAthenzTenantAdmin(identity, tenant.getAthensDomain().get()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java new file mode 100644 index 00000000000..8063a0d1c2d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -0,0 +1,217 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.google.inject.Inject; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.restapi.Path; +import com.yahoo.vespa.hosted.controller.restapi.application.ApplicationInstanceAuthorizer; +import com.yahoo.vespa.hosted.controller.restapi.application.Authorizer; +import com.yahoo.yolean.chain.After; + +import javax.ws.rs.ForbiddenException; +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.NotAuthorizedException; +import javax.ws.rs.WebApplicationException; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; + +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static com.yahoo.jdisc.http.HttpRequest.Method.HEAD; +import static com.yahoo.jdisc.http.HttpRequest.Method.OPTIONS; +import static com.yahoo.jdisc.http.HttpRequest.Method.POST; +import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; +import static com.yahoo.vespa.hosted.controller.restapi.filter.SecurityFilterUtils.sendErrorResponse; + +/** + * A security filter protects all controller apis. + * + * @author bjorncs + */ +@After("com.yahoo.vespa.hosted.controller.athenz.filter.UserAuthWithAthenzPrincipalFilter") +public class ControllerAuthorizationFilter implements SecurityRequestFilter { + + private static final List<Method> WHITELISTED_METHODS = Arrays.asList(GET, OPTIONS, HEAD); + + private final AthenzClientFactory clientFactory; + private final Controller controller; + private final Authorizer authorizer; + private final ApplicationInstanceAuthorizer applicationInstanceAuthorizer; + private final AuthorizationResponseHandler authorizationResponseHandler; + + public interface AuthorizationResponseHandler { + void handle(ResponseHandler responseHandler, WebApplicationException verificationException); + } + + @Inject + public ControllerAuthorizationFilter(AthenzClientFactory clientFactory, + Controller controller, + EntityService entityService, + ZoneRegistry zoneRegistry) { + this(clientFactory, controller, entityService, zoneRegistry, new LoggingAuthorizationResponseHandler()); + } + + ControllerAuthorizationFilter(AthenzClientFactory clientFactory, + Controller controller, + EntityService entityService, + ZoneRegistry zoneRegistry, + AuthorizationResponseHandler authorizationResponseHandler) { + this.clientFactory = clientFactory; + this.controller = controller; + this.authorizer = new Authorizer(controller, entityService, clientFactory); + this.applicationInstanceAuthorizer = new ApplicationInstanceAuthorizer(zoneRegistry, clientFactory); + this.authorizationResponseHandler = authorizationResponseHandler; + } + + // NOTE: Be aware of the ordering of the path pattern matching. Semantics may change if the patterns are evaluated + // in different order. + @Override + public void filter(DiscFilterRequest request, ResponseHandler handler) { + Method method = getMethod(request); + if (isWhiteListedMethod(method)) return; + + try { + Path path = new Path(request.getRequestURI()); + AthenzPrincipal principal = getPrincipal(request); + if (isWhiteListedOperation(path, method)) { + // no authz check + } else if (isHostedOperatorOperation(path, method)) { + verifyIsHostedOperator(principal); + } else if (isTenantAdminOperation(path, method)) { + verifyIsTenantAdmin(principal, getTenantId(path)); + } else if (isTenantPipelineOperation(path, method)) { + verifyIsTenantPipelineOperator(principal, getTenantId(path), getApplicationName(path)); + } else { + throw new ForbiddenException("No access control is explicitly declared for this api."); + } + } catch (WebApplicationException e) { + authorizationResponseHandler.handle(handler, e); + } + } + + private static boolean isWhiteListedMethod(Method method) { + return WHITELISTED_METHODS.contains(method); + } + + private static boolean isWhiteListedOperation(Path path, Method method) { + return path.matches("/screwdriver/v1/jobsToRun") || // TODO EOL'ed API, remove this once api is gone + path.matches("/application/v4/user") && method == PUT || // Create user tenant + path.matches("/application/v4/tenant/{tenant}") && method == POST || // Create tenant + path.matches("/screwdriver/v1/jobreport"); // TODO To be migrated to application/v4 + } + + private static boolean isHostedOperatorOperation(Path path, Method method) { + if (isWhiteListedOperation(path, method)) return false; + return path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying") || + path.matches("/controller/v1/{*}") || + path.matches("/provision/v2/{*}") || + path.matches("/screwdriver/v1/trigger/tenant/{*}") || + path.matches("/zone/v2/{*}"); + } + + private static boolean isTenantAdminOperation(Path path, Method method) { + if (isHostedOperatorOperation(path, method)) return false; + return path.matches("/application/v4/tenant/{tenant}") || + path.matches("/application/v4/tenant/{tenant}/migrateTenantToAthens") || + path.matches("/application/v4/tenant/{tenant}/application/{application}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/dev/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/perf/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override"); + } + + private static boolean isTenantPipelineOperation(Path path, Method method) { + if (isTenantAdminOperation(path, method)) return false; + return path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/prod/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/test/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/staging/{*}"); + } + + private void verifyIsHostedOperator(AthenzPrincipal principal) { + if (!isHostedOperator(principal.getIdentity())) { + throw new ForbiddenException("Vespa operator role required"); + } + } + + private void verifyIsTenantAdmin(AthenzPrincipal principal, TenantId tenantId) { + if (!isTenantAdmin(principal.getIdentity(), tenantId)) { + throw new ForbiddenException("Tenant admin or Vespa operator role required"); + } + } + + private void verifyIsTenantPipelineOperator(AthenzPrincipal principal, + TenantId tenantId, + ApplicationName applicationName) { + controller.tenants().tenant(tenantId) + .ifPresent(tenant -> applicationInstanceAuthorizer.throwIfUnauthorized(principal, tenant, applicationName)); + } + + private boolean isHostedOperator(AthenzIdentity identity) { + return clientFactory.createZmsClientWithServicePrincipal() + .hasHostedOperatorAccess(identity); + } + + private boolean isTenantAdmin(AthenzIdentity identity, TenantId tenantId) { + return controller.tenants().tenant(tenantId) + .map(tenant -> authorizer.isTenantAdmin(identity, tenant)) + .orElse(false); + } + + private static TenantId getTenantId(Path path) { + if (!path.matches("/application/v4/tenant/{tenant}/{*}")) + throw new InternalServerErrorException("Unable to handle path: " + path.asString()); + return new TenantId(path.get("tenant")); + } + + private static ApplicationName getApplicationName(Path path) { + if (!path.matches("/application/v4/tenant/{tenant}/application/{application}/{*}")) + throw new InternalServerErrorException("Unable to handle path: " + path.asString()); + return ApplicationName.from(path.get("application")); + } + + private static Method getMethod(DiscFilterRequest request) { + return Method.valueOf(request.getMethod().toUpperCase()); + } + + private static AthenzPrincipal getPrincipal(DiscFilterRequest request) { + return Optional.ofNullable(request.getUserPrincipal()) + .map(AthenzPrincipal.class::cast) + .orElseThrow(() -> new NotAuthorizedException("User not authenticated")); + } + + private static class LoggingAuthorizationResponseHandler implements AuthorizationResponseHandler { + + @SuppressWarnings("LoggerInitializedWithForeignClass") + private static final Logger log = Logger.getLogger(ControllerAuthorizationFilter.class.getName()); + + @Override + public void handle(ResponseHandler responseHandler, WebApplicationException exception) { + log.log(LogLevel.WARNING, + String.format("Access denied (%d): %s", + exception.getResponse().getStatus(), exception.getMessage())); + } + } + + // TODO Use this as default once we are confident that the access control does not block legal operations + @SuppressWarnings("unused") + static class HttpRespondingAuthorizationResponseHandler implements AuthorizationResponseHandler { + @Override + public void handle(ResponseHandler responseHandler, WebApplicationException exception) { + sendErrorResponse(responseHandler, exception.getResponse().getStatus(), exception.getMessage()); + } + } + + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/SecurityFilterUtils.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SecurityFilterUtils.java index 8e193d3848f..7e7cf7a575d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/SecurityFilterUtils.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SecurityFilterUtils.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.filter; +package com.yahoo.vespa.hosted.controller.restapi.filter; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -12,12 +12,12 @@ import com.yahoo.jdisc.handler.ResponseHandler; /** * @author bjorncs */ -class SecurityFilterUtils { +public class SecurityFilterUtils { private static final ObjectMapper mapper = new ObjectMapper(); private SecurityFilterUtils() {} - static void sendErrorResponse(ResponseHandler responseHandler, int statusCode, String message) { + public static void sendErrorResponse(ResponseHandler responseHandler, int statusCode, String message) { Response response = new Response(statusCode); response.headers().put("Content-Type", "application/json"); ObjectNode errorMessage = mapper.createObjectNode(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index ccc1798358d..691a5ef223d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -4,17 +4,15 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.slime.Slime; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob; -import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; @@ -23,11 +21,14 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; +import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; import com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockOrganization; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; @@ -63,31 +64,33 @@ public final class ControllerTester { private final MemoryNameService nameService; private final RotationsConfig rotationsConfig; private final ArtifactRepositoryMock artifactRepository; + private final EntityService entityService; private Controller controller; public ControllerTester() { this(new MemoryControllerDb(), new AthenzDbMock(), new ManualClock(), new ConfigServerClientMock(), new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), defaultRotationsConfig(), - new MemoryNameService(), new ArtifactRepositoryMock()); + new MemoryNameService(), new ArtifactRepositoryMock(), new MemoryEntityService()); } public ControllerTester(ManualClock clock) { this(new MemoryControllerDb(), new AthenzDbMock(), clock, new ConfigServerClientMock(), new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), defaultRotationsConfig(), - new MemoryNameService(), new ArtifactRepositoryMock()); + new MemoryNameService(), new ArtifactRepositoryMock(), new MemoryEntityService()); } public ControllerTester(RotationsConfig rotationsConfig) { this(new MemoryControllerDb(), new AthenzDbMock(), new ManualClock(), new ConfigServerClientMock(), new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), rotationsConfig, new MemoryNameService(), - new ArtifactRepositoryMock()); + new ArtifactRepositoryMock(), new MemoryEntityService()); } private ControllerTester(ControllerDb db, AthenzDbMock athenzDb, ManualClock clock, ConfigServerClientMock configServer, ZoneRegistryMock zoneRegistry, GitHubMock gitHub, CuratorDb curator, RotationsConfig rotationsConfig, - MemoryNameService nameService, ArtifactRepositoryMock artifactRepository) { + MemoryNameService nameService, ArtifactRepositoryMock artifactRepository, + EntityService entityService) { this.db = db; this.athenzDb = athenzDb; this.clock = clock; @@ -98,8 +101,9 @@ public final class ControllerTester { this.nameService = nameService; this.rotationsConfig = rotationsConfig; this.artifactRepository = artifactRepository; + this.entityService = entityService; this.controller = createController(db, curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, - athenzDb, nameService, artifactRepository); + athenzDb, nameService, artifactRepository, entityService); } public Controller controller() { return controller; } @@ -120,10 +124,12 @@ public final class ControllerTester { public ArtifactRepositoryMock artifactRepository() { return artifactRepository; } + public EntityService entityService() { return entityService; } + /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */ public final void createNewController() { controller = createController(db, curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, athenzDb, - nameService, artifactRepository); + nameService, artifactRepository, entityService); } /** Creates the given tenant and application and deploys it */ @@ -233,12 +239,12 @@ public final class ControllerTester { ConfigServerClientMock configServerClientMock, ManualClock clock, GitHubMock gitHubClientMock, ZoneRegistryMock zoneRegistryMock, AthenzDbMock athensDb, MemoryNameService nameService, - ArtifactRepository artifactRepository) { + ArtifactRepository artifactRepository, EntityService entityService) { Controller controller = new Controller(db, curator, rotationsConfig, gitHubClientMock, - new MemoryEntityService(), + entityService, new MockOrganization(clock), new MemoryGlobalRoutingService(), zoneRegistryMock, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java new file mode 100644 index 00000000000..87215a595a6 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -0,0 +1,193 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzPrincipal; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.HostedAthenzIdentities; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; +import com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter.HttpRespondingAuthorizationResponseHandler; +import org.junit.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Optional; + +import static com.yahoo.container.jdisc.RequestHandlerTestDriver.MockResponseHandler; +import static com.yahoo.jdisc.http.HttpRequest.Method.DELETE; +import static com.yahoo.jdisc.http.HttpRequest.Method.POST; +import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; +import static com.yahoo.jdisc.http.HttpResponse.Status.FORBIDDEN; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author bjorncs + */ +public class ControllerAuthorizationFilterTest { + private static final ObjectMapper mapper = new ObjectMapper(); + + private static final AthenzUser USER = user("john"); + private static final AthenzUser HOSTED_OPERATOR = user("hosted-operator"); + private static final AthenzDomain TENANT_DOMAIN = new AthenzDomain("tenantdomain"); + private static final AthenzService TENANT_ADMIN = new AthenzService(TENANT_DOMAIN, "adminservice"); + private static final AthenzService TENANT_PIPELINE = HostedAthenzIdentities.from(new ScrewdriverId("12345")); + private static final TenantId TENANT = new TenantId("mytenant"); + private static final ApplicationId APPLICATION = new ApplicationId("myapp"); + + @Test + public void white_listed_operations_are_allowed() { + ControllerAuthorizationFilter filter = createFilter(new ControllerTester()); + assertIsAllowed(invokeFilter(filter, createRequest(PUT, "/application/v4/user", USER))); + assertIsAllowed(invokeFilter(filter, createRequest(POST, "/application/v4/tenant/john", USER))); + assertIsAllowed(invokeFilter(filter, createRequest(DELETE, "/screwdriver/v1/jobsToRun", USER))); + assertIsAllowed(invokeFilter(filter, createRequest(DELETE, "/screwdriver/v1/jobreport", USER))); + } + + @Test + public void only_hosted_operator_can_access_operator_apis() { + ControllerTester controllerTester = new ControllerTester(); + controllerTester.athenzDb().hostedOperators.add(HOSTED_OPERATOR); + + ControllerAuthorizationFilter filter = createFilter(controllerTester); + { + String path = "/application/v4/tenant/mytenant/application/myapp/deploying"; + Method method = PUT; + assertIsAllowed(invokeFilter(filter, createRequest(method, path, HOSTED_OPERATOR))); + assertIsForbidden(invokeFilter(filter, createRequest(method, path, USER))); + } + { + String path = "/screwdriver/v1/trigger/tenant/mytenant/application/myapp/"; + Method method = POST; + assertIsAllowed(invokeFilter(filter, createRequest(method, path, HOSTED_OPERATOR))); + assertIsForbidden(invokeFilter(filter, createRequest(method, path, USER))); + } + { + String path = "/provision/v2/provision/enqueue"; + Method method = DELETE; + assertIsAllowed(invokeFilter(filter, createRequest(method, path, HOSTED_OPERATOR))); + assertIsForbidden(invokeFilter(filter, createRequest(method, path, USER))); + } + } + + @Test + public void only_hosted_operator_or_tenant_admin_can_access_tenant_admin_apis() { + ControllerTester controllerTester = new ControllerTester(); + controllerTester.athenzDb().hostedOperators.add(HOSTED_OPERATOR); + controllerTester.createTenant(TENANT.id(), TENANT_DOMAIN.getName(), null); + controllerTester.athenzDb().domains.get(TENANT_DOMAIN).admins.add(TENANT_ADMIN); + + ControllerAuthorizationFilter filter = createFilter(controllerTester); + { + String path = "/application/v4/tenant/mytenant"; + Method method = DELETE; + assertIsAllowed(invokeFilter(filter, createRequest(method, path, HOSTED_OPERATOR))); + assertIsAllowed(invokeFilter(filter, createRequest(method, path, TENANT_ADMIN))); + assertIsForbidden(invokeFilter(filter, createRequest(method, path, USER))); + } + { + String path = "/application/v4/tenant/mytenant/application/myapp/environment/perf/region/myregion/instance/default/deploy"; + Method method = POST; + assertIsAllowed(invokeFilter(filter, createRequest(method, path, HOSTED_OPERATOR))); + assertIsAllowed(invokeFilter(filter, createRequest(method, path, TENANT_ADMIN))); + assertIsForbidden(invokeFilter(filter, createRequest(method, path, USER))); + } + { + String path = "/application/v4/tenant/mytenant/application/myapp/environment/prod/region/myregion/instance/default/global-rotation/override"; + Method method = PUT; + assertIsAllowed(invokeFilter(filter, createRequest(method, path, HOSTED_OPERATOR))); + assertIsAllowed(invokeFilter(filter, createRequest(method, path, TENANT_ADMIN))); + assertIsForbidden(invokeFilter(filter, createRequest(method, path, USER))); + } + } + + @Test + public void only_hosted_operator_and_screwdriver_project_with_deploy_role_can_access_tenant_pipeline_apis() { + ControllerTester controllerTester = new ControllerTester(); + controllerTester.athenzDb().hostedOperators.add(HOSTED_OPERATOR); + controllerTester.createTenant(TENANT.id(), TENANT_DOMAIN.getName(), null); + controllerTester.createApplication(TENANT, APPLICATION.id(), "default", 12345); + AthenzDbMock.Domain domainMock = controllerTester.athenzDb().domains.get(TENANT_DOMAIN); + domainMock.admins.add(TENANT_ADMIN); + domainMock.applications.get(APPLICATION).addRoleMember(ApplicationAction.deploy, TENANT_PIPELINE); + + ControllerAuthorizationFilter filter = createFilter(controllerTester); + { + String path = "/application/v4/tenant/mytenant/application/myapp/environment/prod/region/myregion/instance/default/deploy"; + Method method = POST; + assertIsAllowed(invokeFilter(filter, createRequest(method, path, HOSTED_OPERATOR))); + assertIsAllowed(invokeFilter(filter, createRequest(method, path, TENANT_PIPELINE))); + assertIsForbidden(invokeFilter(filter, createRequest(method, path, TENANT_ADMIN))); + assertIsForbidden(invokeFilter(filter, createRequest(method, path, USER))); + } + } + + private static void assertIsAllowed(Optional<AuthorizationResponse> response) { + assertFalse("Expected no response from filter", response.isPresent()); + } + + private static void assertIsForbidden(Optional<AuthorizationResponse> response) { + assertTrue("Expected a response from filter", response.isPresent()); + assertEquals("Invalid status code", response.get().statusCode, FORBIDDEN); + } + + private static ControllerAuthorizationFilter createFilter(ControllerTester controllerTester) { + return new ControllerAuthorizationFilter(new AthenzClientFactoryMock(controllerTester.athenzDb()), + controllerTester.controller(), + controllerTester.entityService(), + controllerTester.zoneRegistry(), + new HttpRespondingAuthorizationResponseHandler()); + } + + private static Optional<AuthorizationResponse> invokeFilter(ControllerAuthorizationFilter filter, + DiscFilterRequest request) { + MockResponseHandler responseHandlerMock = new MockResponseHandler(); + filter.filter(request, responseHandlerMock); + return Optional.ofNullable(responseHandlerMock.getResponse()) + .map(response -> new AuthorizationResponse(response.getStatus(), getErrorMessage(responseHandlerMock))); + } + + private static DiscFilterRequest createRequest(Method method, String path, AthenzIdentity identity) { + DiscFilterRequest request = mock(DiscFilterRequest.class); + when(request.getMethod()).thenReturn(method.name()); + when(request.getRequestURI()).thenReturn(path); + when(request.getUserPrincipal()).thenReturn(new AthenzPrincipal(identity)); + return request; + } + + private static String getErrorMessage(MockResponseHandler responseHandler) { + try { + return mapper.readTree(responseHandler.readAll()).get("message").asText(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static AthenzUser user(String name) { + return new AthenzUser(name); + } + + private static class AuthorizationResponse { + final int statusCode; + final String message; + + AuthorizationResponse(int statusCode, String message) { + this.statusCode = statusCode; + this.message = message; + } + } +}
\ No newline at end of file diff --git a/dist/vespa.spec b/dist/vespa.spec index 5c1703bbdbe..ff8b292e627 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -99,7 +99,6 @@ Requires: gdb Requires: net-tools %if 0%{?centos} Requires: llvm3.9 -Requires: devtoolset-7-gdb Requires: vespa-boost >= 1.59.0-6 %define _extra_link_directory /usr/lib64/llvm3.9/lib;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib %define _extra_include_directory /usr/include/llvm3.9;/opt/vespa-boost/include;/opt/vespa-cppunit/include diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index 00493e3e016..e006d5aca4c 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -45,7 +45,7 @@ public interface Docker { Map<String, Object> getBlkioStats(); } - default boolean networkNATed() { + default boolean networkNPTed() { return false; } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 15e88f4f253..a72865e023a 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -139,7 +139,7 @@ public class DockerImpl implements Docker { } @Override - public boolean networkNATed() { + public boolean networkNPTed() { return config.networkNATed(); } diff --git a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java index bbd244dd1dc..294ffafa7e7 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java @@ -133,6 +133,8 @@ public class AttributeNode implements ExpressionNode { return handler.values; } else if (value instanceof DocumentUpdate) { return Result.INVALID; + } else if (value instanceof DocumentRemove) { + return Result.INVALID; } return Result.FALSE; } diff --git a/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java b/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java index 242413c8d3e..d12e8d20fa9 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java @@ -43,6 +43,9 @@ public class DocumentNode implements ExpressionNode { doct = ((DocumentPut)op).getDocument().getDataType(); } else if (op instanceof DocumentUpdate) { doct = ((DocumentUpdate)op).getDocumentType(); + } else if (op instanceof DocumentRemove) { + DocumentRemove removeOp = (DocumentRemove)op; + return (removeOp.getId().getDocType().equals(type) ? op : Boolean.FALSE); } else { throw new IllegalStateException("Document class '" + op.getClass().getName() + "' is not supported."); } diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 508bf7f0b18..5286cec4644 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -250,6 +250,21 @@ public class DocumentSelectorTestCase extends junit.framework.TestCase { // TODO Fails: assertEquals(Result.TRUE, evaluate("test.hint + 1 > 13", upd)); } + public void testDocumentRemove() throws ParseException { + assertEquals(Result.TRUE, evaluate("test", createRemove("id:ns:test::1"))); + assertEquals(Result.FALSE, evaluate("test", createRemove("id:ns:null::1"))); + assertEquals(Result.FALSE, evaluate("test", createRemove("userdoc:test:1234:1"))); + assertEquals(Result.INVALID, evaluate("test.hint", createRemove("id:ns:test::1"))); + assertEquals(Result.FALSE, evaluate("test.hint", createRemove("id:ns:null::1"))); + assertEquals(Result.INVALID, evaluate("test.hint == 0", createRemove("id:ns:test::1"))); + assertEquals(Result.INVALID, evaluate("test.anything", createRemove("id:ns:test::1"))); + assertEquals(Result.INVALID, evaluate("test and test.hint == 0", createRemove("id:ns:test::1"))); + } + + private DocumentRemove createRemove(String docId) { + return new DocumentRemove(new DocumentId(docId)); + } + public void testInvalidLogic() throws ParseException { DocumentPut put = new DocumentPut(manager.getDocumentType("test"), new DocumentId("doc:scheme:")); DocumentUpdate upd = new DocumentUpdate(manager.getDocumentType("test"), new DocumentId("doc:scheme:")); diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index db7b48cdc3a..410ac539ff4 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -37,6 +37,7 @@ class DocumentSelectParserTest : public CppUnit::TestFixture { CPPUNIT_TEST(testThatComplexFieldValuesHaveCorrectFieldNames); CPPUNIT_TEST(testBodyFieldDetection); CPPUNIT_TEST(testDocumentUpdates); + CPPUNIT_TEST(testDocumentIdsInRemoves); CPPUNIT_TEST(test_syntax_error_reporting); CPPUNIT_TEST(test_operator_precedence); CPPUNIT_TEST(test_token_used_as_ident_preserves_casing); @@ -102,6 +103,7 @@ public: void testDocumentUpdates2(); void testDocumentUpdates3(); void testDocumentUpdates4(); + void testDocumentIdsInRemoves(); void test_syntax_error_reporting(); void test_operator_precedence(); void test_token_used_as_ident_preserves_casing(); @@ -1238,6 +1240,19 @@ void DocumentSelectParserTest::testDocumentUpdates4() PARSEI("-6 % 10 = -6", *_update[0], True); } +void DocumentSelectParserTest::testDocumentIdsInRemoves() +{ + PARSE("testdoctype1", DocumentId("id:ns:testdoctype1::1"), True); + PARSE("testdoctype1", DocumentId("id:ns:null::1"), False); + PARSE("testdoctype1", DocumentId("userdoc:testdoctype1:1234:1"), False); + PARSE("testdoctype1.headerval", DocumentId("id:ns:testdoctype1::1"), Invalid); + // FIXME: Should ideally be False. As long as there always is an AND node with doctype in a selection expression + // we won't end up sending removes using the wrong route. + PARSE("testdoctype1.headerval", DocumentId("id:ns:null::1"), Invalid); + PARSE("testdoctype1.headerval == 0", DocumentId("id:ns:testdoctype1::1"), Invalid); + PARSE("testdoctype1 and testdoctype1.headerval == 0", DocumentId("id:ns:testdoctype1::1"), Invalid); +} + void DocumentSelectParserTest::testUtf8() { createDocs(); diff --git a/document/src/vespa/document/select/doctype.cpp b/document/src/vespa/document/select/doctype.cpp index bbaba0ae7cf..4ecb388f60d 100644 --- a/document/src/vespa/document/select/doctype.cpp +++ b/document/src/vespa/document/select/doctype.cpp @@ -41,7 +41,7 @@ DocType::contains(const Context &context) const _doctype))); } if (context._docId != NULL) { - return ResultList(Result::False); + return ResultList(Result::get((context._docId->getDocType() == _doctype))); } const DocumentUpdate &upd(*context._docUpdate); return ResultList(Result::get( diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java index e313763fd0e..29a9548579e 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java @@ -145,6 +145,14 @@ public class DocumentRouteSelectorPolicy case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: return selector.accepts(((UpdateDocumentMessage)msg).getDocumentUpdate()) != Result.FALSE; + case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: { + RemoveDocumentMessage removeMsg = (RemoveDocumentMessage)msg; + if (removeMsg.getDocumentId().hasDocType()) { + return selector.accepts(removeMsg.getDocumentRemove()) != Result.FALSE; + } else { + return true; + } + } case DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE: BatchDocumentUpdateMessage bdu = (BatchDocumentUpdateMessage)msg; diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java index fba2edf5cfd..9db4d1c0d2e 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java @@ -356,10 +356,48 @@ public class PolicyTestCase { "route[1].feed \"myfeed\"\n]"; } - private void assertDistribution(PolicyTestFrame frame, String id, String expected) { - frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), - new DocumentId(id))))); - frame.assertSelect(Arrays.asList(expected)); + @Test + public void remove_document_messages_are_sent_to_the_route_handling_the_given_document_type() { + PolicyTestFrame frame = createFrameWithTwoRoutes(); + + frame.setMessage(createRemove("id:ns:testdoc::1")); + frame.assertSelect(Arrays.asList("testdoc-route")); + + frame.setMessage(createRemove("id:ns:other::1")); + frame.assertSelect(Arrays.asList("other-route")); + } + + @Test + public void remove_document_messages_with_legacy_document_ids_are_sent_to_all_routes() { + PolicyTestFrame frame = createFrameWithTwoRoutes(); + + frame.setMessage(createRemove("userdoc:testdoc:1234:1")); + frame.assertSelect(Arrays.asList("testdoc-route", "other-route")); + + frame.setMessage(createRemove("userdoc:other:1234:1")); + frame.assertSelect(Arrays.asList("testdoc-route", "other-route")); + } + + private PolicyTestFrame createFrameWithTwoRoutes() { + PolicyTestFrame result = new PolicyTestFrame(manager); + result.setHop(new HopSpec("test", createDocumentRouteSelectorConfigWithTwoRoutes()) + .addRecipient("testdoc-route").addRecipient("other-route")); + return result; + } + + private String createDocumentRouteSelectorConfigWithTwoRoutes() { + return "[DocumentRouteSelector:raw:" + + "route[2]\n" + + "route[0].name \"testdoc-route\"\n" + + "route[0].selector \"testdoc and testdoc.stringfield != '0'\"\n" + + "route[0].feed \"\"\n" + + "route[1].name \"other-route\"\n" + + "route[1].selector \"other and other.intfield != '0'\"\n" + + "route[1].feed \"\"\n]"; + } + + private RemoveDocumentMessage createRemove(String docId) { + return new RemoveDocumentMessage(new DocumentId(docId)); } @Test diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp index 58db3079631..cc598aef786 100644 --- a/documentapi/src/tests/policies/policies_test.cpp +++ b/documentapi/src/tests/policies/policies_test.cpp @@ -63,6 +63,8 @@ public: void testAND(); void testDocumentRouteSelector(); void testDocumentRouteSelectorIgnore(); + void remove_document_messages_are_sent_to_the_route_handling_the_given_document_type(); + void remove_document_messages_with_legacy_document_ids_are_sent_to_all_routes(); void testExternSend(); void testExternMultipleSlobroks(); void testLoadBalancer(); @@ -103,6 +105,8 @@ Test::Main() { testAND(); TEST_FLUSH(); testDocumentRouteSelector(); TEST_FLUSH(); testDocumentRouteSelectorIgnore(); TEST_FLUSH(); + remove_document_messages_are_sent_to_the_route_handling_the_given_document_type(); TEST_FLUSH(); + remove_document_messages_with_legacy_document_ids_are_sent_to_all_routes(); TEST_FLUSH(); testExternSend(); TEST_FLUSH(); testExternMultipleSlobroks(); TEST_FLUSH(); testLoadBalancer(); TEST_FLUSH(); @@ -673,6 +677,62 @@ Test::testDocumentRouteSelectorIgnore() } namespace { + +vespalib::string +createDocumentRouteSelectorConfigWithTwoRoutes() +{ + return "[DocumentRouteSelector:raw:" + "route[2]\n" + "route[0].name \"testdoc-route\"\n" + "route[0].selector \"testdoc and testdoc.stringfield != '0'\"\n" + "route[0].feed \"\"\n" + "route[1].name \"other-route\"\n" + "route[1].selector \"other and other.intfield != '0'\"\n" + "route[1].feed \"\"\n]"; +} + +std::unique_ptr<TestFrame> +createFrameWithTwoRoutes(DocumentTypeRepo::SP repo) +{ + auto result = std::make_unique<TestFrame>(repo); + result->setHop(mbus::HopSpec("test", createDocumentRouteSelectorConfigWithTwoRoutes()) + .addRecipient("testdoc-route").addRecipient("other-route")); + return result; +} + +std::unique_ptr<RemoveDocumentMessage> +makeRemove(vespalib::string docId) +{ + return std::make_unique<RemoveDocumentMessage>(DocumentId(docId)); +} + +} + +void +Test::remove_document_messages_are_sent_to_the_route_handling_the_given_document_type() +{ + auto frame = createFrameWithTwoRoutes(_repo); + + frame->setMessage(makeRemove("id:ns:testdoc::1")); + EXPECT_TRUE(frame->testSelect({"testdoc-route"})); + + frame->setMessage(makeRemove("id:ns:other::1")); + EXPECT_TRUE(frame->testSelect({"other-route"})); +} + +void +Test::remove_document_messages_with_legacy_document_ids_are_sent_to_all_routes() +{ + auto frame = createFrameWithTwoRoutes(_repo); + + frame->setMessage(makeRemove("userdoc:testdoc:1234:1")); + EXPECT_TRUE(frame->testSelect({"testdoc-route", "other-route"})); + + frame->setMessage(makeRemove("userdoc:other:1234:1")); + EXPECT_TRUE(frame->testSelect({"testdoc-route", "other-route"})); +} + +namespace { string getDefaultDistributionConfig( uint16_t redundancy = 2, uint16_t nodeCount = 10, storage::lib::Distribution::DiskDistribution distr diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp index 6756f694267..c763723ada1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp @@ -15,6 +15,8 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> + LOG_SETUP(".documentrouteselectorpolicy"); using document::select::Result; @@ -129,6 +131,15 @@ DocumentRouteSelectorPolicy::select(mbus::RoutingContext &context, const vespali case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: return it->second->contains(static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()) != Result::False; + case DocumentProtocol::MESSAGE_REMOVEDOCUMENT: { + const RemoveDocumentMessage &removeMsg = static_cast<const RemoveDocumentMessage &>(msg); + if (removeMsg.getDocumentId().hasDocType()) { + return it->second->contains(removeMsg.getDocumentId()) != Result::False; + } else { + return true; + } + } + case DocumentProtocol::MESSAGE_MULTIOPERATION: { const MultiOperationMessage& mom = static_cast<const MultiOperationMessage&>(msg); diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 8378af53098..3e0c0a12a4f 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -27,7 +27,6 @@ vespa_define_module( src/tests/tensor/dense_dot_product_function src/tests/tensor/dense_tensor_address_combiner src/tests/tensor/dense_tensor_builder - src/tests/tensor/dense_tensor_function_optimizer src/tests/tensor/dense_xw_product_function src/tests/tensor/vector_from_doubles_function src/tests/tensor/sparse_tensor_builder diff --git a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp index 6b11e2e7034..714eb870b3e 100644 --- a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp +++ b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp @@ -154,156 +154,6 @@ TEST("require that basic addition works") { //----------------------------------------------------------------------------- -struct InnerProduct { - const TensorEngine &engine; - Function function; - TensorSpec a; - TensorSpec b; - TensorSpec expect; - NodeTypes types; - InterpretedFunction interpreted; - ~InnerProduct() {} - InnerProduct(const vespalib::string &expr) - : engine(DefaultTensorEngine::ref()), - function(Function::parse({"a", "b"}, expr)), - a("null"), b("null"), expect("null"), - types(), - interpreted(engine, function, types) {} - InnerProduct(const vespalib::string &expr, - TensorSpec a_in, - TensorSpec b_in, - TensorSpec expect_in) - : engine(DefaultTensorEngine::ref()), - function(Function::parse(expr)), - a(a_in), b(b_in), expect(expect_in), - types(function, {ValueType::from_spec(a.type()), ValueType::from_spec(b.type())}), - interpreted(engine, function, types) {} - void verify_optimized() const { - EXPECT_LESS(interpreted.program_size(), 4u); - InterpretedFunction::Context ctx(interpreted); - Value::UP va = engine.from_spec(a); - Value::UP vb = engine.from_spec(b); - SimpleObjectParams params({*va,*vb}); - const Value &result = interpreted.eval(ctx, params); - EXPECT_EQUAL(engine.to_spec(result), expect); - } - void verify_not_optimized() const { - EXPECT_EQUAL(4u, interpreted.program_size()); - } -}; - -struct UntypedIP : InnerProduct { - UntypedIP(const vespalib::string &expr) : InnerProduct(expr) { - a = TensorSpec("double").add({}, 2.0); - b = TensorSpec("double").add({}, 3.0); - expect = TensorSpec("double").add({}, 6.0); - } -}; - -struct DotProduct : InnerProduct { - DotProduct(const vespalib::string &expr) - : InnerProduct(expr, - TensorSpec("tensor(x[3])") - .add({{"x", 0}}, 5.0) - .add({{"x", 1}}, 3.0) - .add({{"x", 2}}, 2.0), - TensorSpec("tensor(x[3])") - .add({{"x", 0}}, 7.0) - .add({{"x", 1}}, 11.0) - .add({{"x", 2}}, 13.0), - TensorSpec("double") - .add({}, (5.0 * 7.0) + (3.0 * 11.0) + (2.0 * 13.0))) {} -}; - -struct XW : InnerProduct { - XW(const vespalib::string &expr) - : InnerProduct(expr, - TensorSpec("tensor(x[2])") - .add({{"x", 0}}, 1.0) - .add({{"x", 1}}, 2.0), - TensorSpec("tensor(x[2],y[3])") - .add({{"y", 0},{"x", 0}}, 3.0) - .add({{"y", 0},{"x", 1}}, 5.0) - .add({{"y", 1},{"x", 0}}, 7.0) - .add({{"y", 1},{"x", 1}}, 11.0) - .add({{"y", 2},{"x", 0}}, 13.0) - .add({{"y", 2},{"x", 1}}, 17.0), - TensorSpec("tensor(y[3])") - .add({{"y", 0}}, (1.0 * 3.0) + (2.0 * 5.0)) - .add({{"y", 1}}, (1.0 * 7.0) + (2.0 * 11.0)) - .add({{"y", 2}}, (1.0 * 13.0) + (2.0 * 17.0))) {} -}; - -struct MatMul : InnerProduct { - MatMul(const vespalib::string &expr) - : InnerProduct(expr, - TensorSpec("tensor(x[2],y[2])") - .add({{"x", 0},{"y", 0}}, 1.0) - .add({{"x", 0},{"y", 1}}, 2.0) - .add({{"x", 1},{"y", 0}}, 3.0) - .add({{"x", 1},{"y", 1}}, 5.0), - TensorSpec("tensor(y[2],z[2])") - .add({{"y", 0},{"z", 0}}, 7.0) - .add({{"y", 0},{"z", 1}}, 11.0) - .add({{"y", 1},{"z", 0}}, 13.0) - .add({{"y", 1},{"z", 1}}, 17.0), - TensorSpec("tensor(x[2],z[2])") - .add({{"x", 0},{"z", 0}}, (1.0 * 7.0) + (2.0 * 13.0)) - .add({{"x", 0},{"z", 1}}, (1.0 * 11.0) + (2.0 * 17.0)) - .add({{"x", 1},{"z", 0}}, (3.0 * 7.0) + (5.0 * 13.0)) - .add({{"x", 1},{"z", 1}}, (3.0 * 11.0) + (5.0 * 17.0))) {} -}; - -TEST("require that inner product is not optimized for unknown types") { - TEST_DO(UntypedIP("reduce(a*b,sum)").verify_not_optimized()); - TEST_DO(UntypedIP("reduce(join(a,b,f(x,y)(x*y)),sum)").verify_not_optimized()); -} - -TEST("require that dot product works with tensor function") { - TEST_DO(DotProduct("reduce(a*b,sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(x*y)),sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(b*a,sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(b,a,f(x,y)(x*y)),sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(y*x)),sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(b,a,f(x,y)(y*x)),sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(a*b,sum,x)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(x*y)),sum,x)").verify_optimized()); - TEST_DO(DotProduct("reduce(b*a,sum,x)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(b,a,f(x,y)(x*y)),sum,x)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(y*x)),sum,x)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(b,a,f(x,y)(y*x)),sum,x)").verify_optimized()); -} - -TEST("require that vector matrix multiplication works with tensor function") { - TEST_DO(XW("reduce(a*b,sum,x)").verify_optimized()); - TEST_DO(XW("reduce(join(a,b,f(x,y)(x*y)),sum,x)").verify_optimized()); - TEST_DO(XW("reduce(b*a,sum,x)").verify_optimized()); - TEST_DO(XW("reduce(join(b,a,f(x,y)(x*y)),sum,x)").verify_optimized()); - TEST_DO(XW("reduce(join(a,b,f(x,y)(y*x)),sum,x)").verify_optimized()); - TEST_DO(XW("reduce(join(b,a,f(x,y)(y*x)),sum,x)").verify_optimized()); -} - -TEST("require that matrix multiplication is not optimized (yet)") { - TEST_DO(MatMul("reduce(a*b,sum,y)").verify_not_optimized()); - TEST_DO(MatMul("reduce(join(a,b,f(x,y)(x*y)),sum,y)").verify_not_optimized()); - TEST_DO(MatMul("reduce(b*a,sum,y)").verify_not_optimized()); - TEST_DO(MatMul("reduce(join(b,a,f(x,y)(x*y)),sum,y)").verify_not_optimized()); - TEST_DO(MatMul("reduce(join(a,b,f(x,y)(y*x)),sum,y)").verify_not_optimized()); - TEST_DO(MatMul("reduce(join(b,a,f(x,y)(y*x)),sum,y)").verify_not_optimized()); -} - -TEST("require that expressions similar to inner product are not optimized") { - TEST_DO(DotProduct("reduce(a*b,prod)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(a*b,max)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(a+b,sum)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(x+y)),sum)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(x*x)),sum)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(y*y)),sum)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(x*y*1)),sum)").verify_not_optimized()); -} - -//----------------------------------------------------------------------------- - TEST("require that functions with non-compilable lambdas cannot be interpreted") { auto good_map = Function::parse("map(a,f(x)(x+1))"); auto good_join = Function::parse("join(a,b,f(x,y)(x+y))"); diff --git a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp index 71bbacc7806..fb48e445180 100644 --- a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp @@ -7,6 +7,8 @@ #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/tensor/dense/dense_tensor_builder.h> #include <vespa/eval/tensor/dense/dense_tensor_view.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/eval_fixture.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/stash.h> @@ -15,128 +17,68 @@ LOG_SETUP("dense_dot_product_function_test"); using namespace vespalib; using namespace vespalib::eval; +using namespace vespalib::eval::test; using namespace vespalib::tensor; -tensor::Tensor::UP -makeTensor(size_t numCells, double cellBias) -{ - DenseTensorBuilder builder; - DenseTensorBuilder::Dimension dim = builder.defineDimension("x", numCells); - for (size_t i = 0; i < numCells; ++i) { - builder.addLabel(dim, i).addCell(i + cellBias); - } - return builder.build(); +const TensorEngine &prod_engine = DefaultTensorEngine::ref(); + +struct MyVecSeq : Sequence { + double bias; + double operator[](size_t i) const override { return (i + bias); } + MyVecSeq(double cellBias) : bias(cellBias) {} +}; + +TensorSpec makeTensor(size_t numCells, double cellBias) { + return spec({x(numCells)}, MyVecSeq(cellBias)); } -double -calcDotProduct(const DenseTensor &lhs, const DenseTensor &rhs) -{ - size_t numCells = std::min(lhs.cellsRef().size(), rhs.cellsRef().size()); +const double leftBias = 3.0; +const double rightBias = 5.0; + +double calcDotProduct(size_t numCells) { double result = 0; for (size_t i = 0; i < numCells; ++i) { - result += (lhs.cellsRef()[i] * rhs.cellsRef()[i]); + result += (i + leftBias) * (i + rightBias); } return result; } -const DenseTensor & -asDenseTensor(const tensor::Tensor &tensor) -{ - return dynamic_cast<const DenseTensor &>(tensor); -} - -class FunctionInput -{ -private: - tensor::Tensor::UP _lhsTensor; - tensor::Tensor::UP _rhsTensor; - const DenseTensor &_lhsDenseTensor; - const DenseTensor &_rhsDenseTensor; - std::vector<Value::CREF> _params; - -public: - FunctionInput(size_t lhsNumCells, size_t rhsNumCells) - : _lhsTensor(makeTensor(lhsNumCells, 3.0)), - _rhsTensor(makeTensor(rhsNumCells, 5.0)), - _lhsDenseTensor(asDenseTensor(*_lhsTensor)), - _rhsDenseTensor(asDenseTensor(*_rhsTensor)) - { - _params.emplace_back(_lhsDenseTensor); - _params.emplace_back(_rhsDenseTensor); - } - SimpleObjectParams get() const { return SimpleObjectParams(_params); } - const Value ¶m(size_t idx) const { return _params[idx]; } - double expectedDotProduct() const { - return calcDotProduct(_lhsDenseTensor, _rhsDenseTensor); - } +void check_gen_with_result(size_t l, size_t r, double wanted) { + EvalFixture::ParamRepo param_repo; + param_repo.add("a", makeTensor(l, leftBias)); + param_repo.add("b", makeTensor(r, rightBias)); + vespalib::string expr = "reduce(a*b,sum,x)"; + EvalFixture evaluator(prod_engine, expr, param_repo, true); + EXPECT_EQUAL(spec(wanted), evaluator.result()); + EXPECT_EQUAL(evaluator.result(), EvalFixture::ref(expr, param_repo)); + auto info = evaluator.find_all<DenseDotProductFunction>(); + EXPECT_EQUAL(info.size(), 1u); }; -struct Fixture -{ - FunctionInput input; - tensor_function::Inject a; - tensor_function::Inject b; - DenseDotProductFunction function; - Fixture(size_t lhsNumCells, size_t rhsNumCells); - ~Fixture(); - double eval() const { - InterpretedFunction ifun(DefaultTensorEngine::ref(), function); - InterpretedFunction::Context ictx(ifun); - const Value &result = ifun.eval(ictx, input.get()); - ASSERT_TRUE(result.is_double()); - LOG(info, "eval(): (%s) * (%s) = %f", - input.param(0).type().to_spec().c_str(), - input.param(1).type().to_spec().c_str(), - result.as_double()); - return result.as_double(); - } -}; - -Fixture::Fixture(size_t lhsNumCells, size_t rhsNumCells) - : input(lhsNumCells, rhsNumCells), - a(input.param(0).type(), 0), - b(input.param(1).type(), 1), - function(a, b) -{ } - -Fixture::~Fixture() { } - -void -assertDotProduct(size_t numCells) -{ - Fixture f(numCells, numCells); - EXPECT_EQUAL(f.input.expectedDotProduct(), f.eval()); -} +// this should not be possible to set up: +// TEST("require that empty dot product is correct") -void -assertDotProduct(size_t lhsNumCells, size_t rhsNumCells) -{ - Fixture f(lhsNumCells, rhsNumCells); - EXPECT_EQUAL(f.input.expectedDotProduct(), f.eval()); +TEST("require that basic dot product with equal sizes is correct") { + check_gen_with_result(2, 2, (3.0 * 5.0) + (4.0 * 6.0)); } -TEST_F("require that empty dot product is correct", Fixture(0, 0)) -{ - EXPECT_EQUAL(0.0, f.eval()); +TEST("require that basic dot product with un-equal sizes is correct") { + check_gen_with_result(2, 3, (3.0 * 5.0) + (4.0 * 6.0)); + check_gen_with_result(3, 2, (3.0 * 5.0) + (4.0 * 6.0)); } -TEST_F("require that basic dot product with equal sizes is correct", Fixture(2, 2)) -{ - EXPECT_EQUAL((3.0 * 5.0) + (4.0 * 6.0), f.eval()); -} +//----------------------------------------------------------------------------- -TEST_F("require that basic dot product with un-equal sizes is correct", Fixture(2, 3)) -{ - EXPECT_EQUAL((3.0 * 5.0) + (4.0 * 6.0), f.eval()); +void assertDotProduct(size_t numCells) { + check_gen_with_result(numCells, numCells, calcDotProduct(numCells)); } -TEST_F("require that basic dot product with un-equal sizes is correct", Fixture(3, 2)) -{ - EXPECT_EQUAL((3.0 * 5.0) + (4.0 * 6.0), f.eval()); +void assertDotProduct(size_t lhsNumCells, size_t rhsNumCells) { + size_t numCells = std::min(lhsNumCells, rhsNumCells); + check_gen_with_result(lhsNumCells, rhsNumCells, calcDotProduct(numCells)); } -TEST("require that dot product with equal sizes is correct") -{ +TEST("require that dot product with equal sizes is correct") { TEST_DO(assertDotProduct(8)); TEST_DO(assertDotProduct(16)); TEST_DO(assertDotProduct(32)); @@ -156,9 +98,9 @@ TEST("require that dot product with equal sizes is correct") TEST_DO(assertDotProduct(1024 + 3)); } -TEST("require that dot product with un-equal sizes is correct") -{ +TEST("require that dot product with un-equal sizes is correct") { TEST_DO(assertDotProduct(8, 8 + 3)); + TEST_DO(assertDotProduct(8 + 3, 8)); TEST_DO(assertDotProduct(16, 16 + 3)); TEST_DO(assertDotProduct(32, 32 + 3)); TEST_DO(assertDotProduct(64, 64 + 3)); @@ -168,4 +110,78 @@ TEST("require that dot product with un-equal sizes is correct") TEST_DO(assertDotProduct(1024, 1024 + 3)); } +//----------------------------------------------------------------------------- + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add("v01_x1", spec({x(1)}, MyVecSeq(2.0))) + .add("v02_x3", spec({x(3)}, MyVecSeq(4.0))) + .add("v03_x3", spec({x(3)}, MyVecSeq(5.0))) + .add("v04_y3", spec({y(3)}, MyVecSeq(10))) + .add("v05_x5", spec({x(5)}, MyVecSeq(6.0))) + .add("v06_x5", spec({x(5)}, MyVecSeq(7.0))) + .add("v07_x3_a", spec({x(3)}, MyVecSeq(8.0)), "any") + .add("v08_x3_u", spec({x(3)}, MyVecSeq(9.0)), "tensor(x[])") + .add("v09_x4_u", spec({x(4)}, MyVecSeq(3.0)), "tensor(x[])") + .add("m01_x3y3", spec({x(3),y(3)}, MyVecSeq(0))); +} +EvalFixture::ParamRepo param_repo = make_params(); + +void assertOptimized(const vespalib::string &expr) { + EvalFixture fixture(prod_engine, expr, param_repo, true); + EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); + auto info = fixture.find_all<DenseDotProductFunction>(); + EXPECT_EQUAL(info.size(), 1u); +} + +void assertNotOptimized(const vespalib::string &expr) { + EvalFixture fixture(prod_engine, expr, param_repo, true); + EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); + auto info = fixture.find_all<DenseDotProductFunction>(); + EXPECT_TRUE(info.empty()); +} + +TEST("require that dot product is not optimized for unknown types") { + TEST_DO(assertNotOptimized("reduce(v02_x3*v07_x3_a,sum)")); + TEST_DO(assertNotOptimized("reduce(v07_x3_a*v03_x3,sum)")); +} + +TEST("require that dot product works with tensor function") { + TEST_DO(assertOptimized("reduce(v05_x5*v06_x5,sum)")); + TEST_DO(assertOptimized("reduce(v05_x5*v06_x5,sum,x)")); + TEST_DO(assertOptimized("reduce(join(v05_x5,v06_x5,f(x,y)(x*y)),sum)")); + TEST_DO(assertOptimized("reduce(join(v05_x5,v06_x5,f(x,y)(x*y)),sum,x)")); +} + +TEST("require that dot product with compatible dimensions is optimized") { + TEST_DO(assertOptimized("reduce(v01_x1*v01_x1,sum)")); + TEST_DO(assertOptimized("reduce(v02_x3*v03_x3,sum)")); + TEST_DO(assertOptimized("reduce(v05_x5*v06_x5,sum)")); + + TEST_DO(assertOptimized("reduce(v02_x3*v06_x5,sum)")); + TEST_DO(assertOptimized("reduce(v05_x5*v03_x3,sum)")); + TEST_DO(assertOptimized("reduce(v08_x3_u*v05_x5,sum)")); + TEST_DO(assertOptimized("reduce(v05_x5*v08_x3_u,sum)")); +} + +TEST("require that dot product with incompatible dimensions is NOT optimized") { + TEST_DO(assertNotOptimized("reduce(v02_x3*v04_y3,sum)")); + TEST_DO(assertNotOptimized("reduce(v04_y3*v02_x3,sum)")); + TEST_DO(assertNotOptimized("reduce(v08_x3_u*v04_y3,sum)")); + TEST_DO(assertNotOptimized("reduce(v04_y3*v08_x3_u,sum)")); + TEST_DO(assertNotOptimized("reduce(v02_x3*m01_x3y3,sum)")); + TEST_DO(assertNotOptimized("reduce(m01_x3y3*v02_x3,sum)")); +} + +TEST("require that expressions similar to dot product are not optimized") { + TEST_DO(assertNotOptimized("reduce(v02_x3*v03_x3,prod)")); + TEST_DO(assertNotOptimized("reduce(v02_x3+v03_x3,sum)")); + TEST_DO(assertNotOptimized("reduce(join(v02_x3,v03_x3,f(x,y)(x+y)),sum)")); + TEST_DO(assertNotOptimized("reduce(join(v02_x3,v03_x3,f(x,y)(x*x)),sum)")); + TEST_DO(assertNotOptimized("reduce(join(v02_x3,v03_x3,f(x,y)(y*y)),sum)")); + // TEST_DO(assertNotOptimized("reduce(join(v02_x3,v03_x3,f(x,y)(y*x)),sum)")); +} + +//----------------------------------------------------------------------------- + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/dense_tensor_function_optimizer/CMakeLists.txt b/eval/src/tests/tensor/dense_tensor_function_optimizer/CMakeLists.txt deleted file mode 100644 index 3a95ef776d7..00000000000 --- a/eval/src/tests/tensor/dense_tensor_function_optimizer/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_dense_tensor_function_optimizer_test_app TEST - SOURCES - dense_tensor_function_optimizer_test.cpp - DEPENDS - vespaeval -) -vespa_add_test(NAME eval_dense_tensor_function_optimizer_test_app COMMAND eval_dense_tensor_function_optimizer_test_app) diff --git a/eval/src/tests/tensor/dense_tensor_function_optimizer/FILES b/eval/src/tests/tensor/dense_tensor_function_optimizer/FILES deleted file mode 100644 index 3c4ec2f1753..00000000000 --- a/eval/src/tests/tensor/dense_tensor_function_optimizer/FILES +++ /dev/null @@ -1 +0,0 @@ -dense_tensor_function_compiler_test.cpp diff --git a/eval/src/tests/tensor/dense_tensor_function_optimizer/dense_tensor_function_optimizer_test.cpp b/eval/src/tests/tensor/dense_tensor_function_optimizer/dense_tensor_function_optimizer_test.cpp deleted file mode 100644 index 269a1e265c5..00000000000 --- a/eval/src/tests/tensor/dense_tensor_function_optimizer/dense_tensor_function_optimizer_test.cpp +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/eval/tensor/dense/dense_dot_product_function.h> -#include <vespa/eval/tensor/dense/dense_xw_product_function.h> -#include <vespa/eval/eval/operation.h> - -using namespace vespalib::eval; -using namespace vespalib::eval::operation; -using namespace vespalib::eval::tensor_function; -using namespace vespalib::tensor; -using vespalib::Stash; - -//----------------------------------------------------------------------------- - -const TensorFunction & -optimizeDotProduct(const vespalib::string &lhsType, - const vespalib::string &rhsType, - Stash &stash) -{ - const Node &reduceNode = reduce(join(inject(ValueType::from_spec(lhsType), 1, stash), - inject(ValueType::from_spec(rhsType), 3, stash), - Mul::f, stash), - Aggr::SUM, {}, stash); - return DenseDotProductFunction::optimize(reduceNode, stash); -} - -void assertParam(const TensorFunction &node, size_t expect_idx) { - auto inject = as<Inject>(node); - ASSERT_TRUE(inject); - EXPECT_EQUAL(inject->param_idx(), expect_idx); -} - -void -assertOptimizedDotProduct(const vespalib::string &lhsType, - const vespalib::string &rhsType) -{ - Stash stash; - const TensorFunction &func = optimizeDotProduct(lhsType, rhsType, stash); - const DenseDotProductFunction *dotProduct = as<DenseDotProductFunction>(func); - ASSERT_TRUE(dotProduct); - TEST_DO(assertParam(dotProduct->lhs(), 1)); - TEST_DO(assertParam(dotProduct->rhs(), 3)); -} - -void -assertNotOptimizedDotProduct(const vespalib::string &lhsType, - const vespalib::string &rhsType) -{ - Stash stash; - const TensorFunction &func = optimizeDotProduct(lhsType, rhsType, stash); - const Reduce *reduce = as<Reduce>(func); - EXPECT_TRUE(reduce); -} - -//----------------------------------------------------------------------------- - -const TensorFunction & -optimizeXWProduct(const vespalib::string &lhsType, - const vespalib::string &rhsType, - const vespalib::string &dim, - Stash &stash) -{ - const Node &reduceNode = reduce(join(inject(ValueType::from_spec(lhsType), 1, stash), - inject(ValueType::from_spec(rhsType), 3, stash), - Mul::f, stash), - Aggr::SUM, {dim}, stash); - return DenseXWProductFunction::optimize(reduceNode, stash); -} - -void -assertOptimizedXWProduct(const vespalib::string &vecTypeStr, - const vespalib::string &matTypeStr, - const vespalib::string &dim) -{ - Stash stash; - const TensorFunction &func = optimizeXWProduct(vecTypeStr, matTypeStr, dim, stash); - const TensorFunction &inv_func = optimizeXWProduct(matTypeStr, vecTypeStr, dim, stash); - const DenseXWProductFunction *xwProduct = as<DenseXWProductFunction>(func); - const DenseXWProductFunction *inv_xwProduct = as<DenseXWProductFunction>(inv_func); - ValueType vecType = ValueType::from_spec(vecTypeStr); - ValueType matType = ValueType::from_spec(matTypeStr); - size_t common_idx = matType.dimension_index(vecType.dimensions()[0].name); - ASSERT_TRUE(xwProduct); - ASSERT_TRUE(inv_xwProduct); - ASSERT_TRUE(common_idx != ValueType::Dimension::npos); - TEST_DO(assertParam(xwProduct->lhs(), 1)); - TEST_DO(assertParam(inv_xwProduct->lhs(), 3)); - TEST_DO(assertParam(xwProduct->rhs(), 3)); - TEST_DO(assertParam(inv_xwProduct->rhs(), 1)); - EXPECT_EQUAL(xwProduct->vectorSize(), vecType.dimensions()[0].size); - EXPECT_EQUAL(inv_xwProduct->vectorSize(), vecType.dimensions()[0].size); - EXPECT_EQUAL(xwProduct->resultSize(), matType.dimensions()[1 - common_idx].size); - EXPECT_EQUAL(inv_xwProduct->resultSize(), matType.dimensions()[1 - common_idx].size); - EXPECT_EQUAL(xwProduct->matrixHasCommonDimensionInnermost(), (common_idx == 1)); - EXPECT_EQUAL(inv_xwProduct->matrixHasCommonDimensionInnermost(), (common_idx == 1)); -} - -void -assertNotOptimizedXWProduct(const vespalib::string &vecType, - const vespalib::string &matType, - const vespalib::string &dim) -{ - Stash stash; - const TensorFunction &func = optimizeXWProduct(vecType, matType, dim, stash); - const TensorFunction &inv_func = optimizeXWProduct(matType, vecType, dim, stash); - const Reduce *reduce = as<Reduce>(func); - const Reduce *inv_reduce = as<Reduce>(inv_func); - EXPECT_TRUE(reduce); - EXPECT_TRUE(inv_reduce); -} - -//----------------------------------------------------------------------------- - -TEST("require that dot product with compatible dimensions is optimized") -{ - TEST_DO(assertOptimizedDotProduct("tensor(x[5])", "tensor(x[5])")); - TEST_DO(assertOptimizedDotProduct("tensor(x[3])", "tensor(x[5])")); - TEST_DO(assertOptimizedDotProduct("tensor(x[5])", "tensor(x[3])")); - TEST_DO(assertOptimizedDotProduct("tensor(x[])", "tensor(x[5])")); - TEST_DO(assertOptimizedDotProduct("tensor(x[5])", "tensor(x[])")); - TEST_DO(assertOptimizedDotProduct("tensor(x[])", "tensor(x[])")); -} - -TEST("require that dot product with incompatible dimensions is NOT optimized") -{ - TEST_DO(assertNotOptimizedDotProduct("tensor(x[5])", "tensor(y[5])")); - TEST_DO(assertNotOptimizedDotProduct("tensor(y[5])", "tensor(x[5])")); - TEST_DO(assertNotOptimizedDotProduct("tensor(y[])", "tensor(x[])")); - TEST_DO(assertNotOptimizedDotProduct("tensor(x[5])", "tensor(x[5],y[7])")); - TEST_DO(assertNotOptimizedDotProduct("tensor(x[5],y[7])", "tensor(x[5],y[7])")); -} - -//----------------------------------------------------------------------------- - -TEST("require that xw products with compatible dimensions are optimized") { - TEST_DO(assertOptimizedXWProduct("tensor(x[3])", "tensor(x[3],y[4])", "x")); - TEST_DO(assertOptimizedXWProduct("tensor(y[4])", "tensor(x[3],y[4])", "y")); -} - -TEST("require that xw products with incompatible dimensions are not optimized") { - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(x[3],y[4])", "y")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[])", "tensor(x[3],y[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(x[],y[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(x[3],y[])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[2])", "tensor(x[3],y[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[4])", "tensor(x[3],y[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(y[3],z[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(y[3],z[4])", "y")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(y[3],z[4])", "z")); - TEST_DO(assertNotOptimizedXWProduct("tensor(y[4])", "tensor(x[3],y[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(y[3])", "tensor(x[3],y[4])", "y")); - TEST_DO(assertNotOptimizedXWProduct("tensor(y[5])", "tensor(x[3],y[4])", "y")); -} - -//----------------------------------------------------------------------------- - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java index 0d98176d631..7627e9f04fb 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java @@ -72,7 +72,7 @@ class FileAcquirerImpl implements FileAcquirer { private void logWarning() { if (logCount == 0 || System.currentTimeMillis() > nextLogTime ) { - log.warning("Could not connect to the file distributor '" + spec.toString() + "'" + " - " + this + "@" + System.identityHashCode(this)); + log.warning("Could not connect to the config proxy '" + spec.toString() + "'" + " - " + this + "@" + System.identityHashCode(this)); nextLogTime = System.currentTimeMillis() + Math.min(TimeUnit.DAYS.toMillis(1), @@ -152,7 +152,7 @@ class FileAcquirerImpl implements FileAcquirer { if (request.checkReturnTypes("s")) { return new File(request.returnValues().get(0).asString()); } else if (!request.isError()) { - throw new RuntimeException("Invalid answer from file distributor: " + request.returnValues()); + throw new RuntimeException("Invalid answer from config proxy: " + request.returnValues()); } else if (temporaryError(request.errorCode())) { log.log(LogLevel.INFO, "Retrying waitFor: " + request.errorCode() + " -- " + request.errorMessage()); Thread.sleep(1000); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java index e9d2e9f7e8a..3e81321f92e 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.filedistribution; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.FileReference; import com.yahoo.jrt.DoubleArray; import com.yahoo.jrt.Int32Value; @@ -15,6 +16,8 @@ import java.io.File; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -30,6 +33,8 @@ public class FileDistributionRpcServer { private final Supervisor supervisor; private final FileDownloader downloader; + private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), + new DaemonThreadFactory("Rpc executor")); public FileDistributionRpcServer(Supervisor supervisor, FileDownloader downloader) { this.supervisor = supervisor; @@ -74,22 +79,7 @@ public class FileDistributionRpcServer { @SuppressWarnings({"UnusedDeclaration"}) public final void getFile(Request req) { req.detach(); - FileReference fileReference = new FileReference(req.parameters().get(0).asString()); - log.log(LogLevel.DEBUG, "getFile() called for file reference '" + fileReference.value() + "'"); - Optional<File> pathToFile = downloader.getFile(fileReference); - try { - if (pathToFile.isPresent()) { - req.returnValues().add(new StringValue(pathToFile.get().getAbsolutePath())); - log.log(LogLevel.DEBUG, "File reference '" + fileReference.value() + "' available at " + pathToFile.get()); - } else { - log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); - req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found"); - } - } catch (Throwable e) { - log.log(LogLevel.WARNING, "File reference '" + fileReference.value() + "' got exception: " + e.getMessage()); - req.setError(fileReferenceInternalError, "File reference '" + fileReference.value() + "' removed"); - } - req.returnRequest(); + rpcDownloadExecutor.execute(() -> downloadFile(req)); } @SuppressWarnings({"UnusedDeclaration"}) @@ -123,4 +113,23 @@ public class FileDistributionRpcServer { req.returnValues().add(new Int32Value(0)); } + private void downloadFile(Request req) { + FileReference fileReference = new FileReference(req.parameters().get(0).asString()); + log.log(LogLevel.DEBUG, "getFile() called for file reference '" + fileReference.value() + "'"); + Optional<File> pathToFile = downloader.getFile(fileReference); + try { + if (pathToFile.isPresent()) { + req.returnValues().add(new StringValue(pathToFile.get().getAbsolutePath())); + log.log(LogLevel.DEBUG, "File reference '" + fileReference.value() + "' available at " + pathToFile.get()); + } else { + log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); + req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found"); + } + } catch (Throwable e) { + log.log(LogLevel.WARNING, "File reference '" + fileReference.value() + "' got exception: " + e.getMessage()); + req.setError(fileReferenceInternalError, "File reference '" + fileReference.value() + "' removed"); + } + req.returnRequest(); + } + } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 20ad2e48fe2..8e230a1a2cc 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -36,7 +36,7 @@ public class FileReferenceDownloader { private final static Duration rpcTimeout = Duration.ofSeconds(10); private final ExecutorService downloadExecutor = - Executors.newFixedThreadPool(10, new DaemonThreadFactory("filereference downloader")); + Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), new DaemonThreadFactory("filereference downloader")); private final ConnectionPool connectionPool; private final Map<FileReference, FileReferenceDownload> downloads = new LinkedHashMap<>(); private final Map<FileReference, Double> downloadStatus = new HashMap<>(); // between 0 and 1 @@ -51,10 +51,6 @@ public class FileReferenceDownloader { private void startDownload(Duration timeout, FileReferenceDownload fileReferenceDownload) { FileReference fileReference = fileReferenceDownload.fileReference(); - synchronized (downloads) { - downloads.put(fileReference, fileReferenceDownload); - downloadStatus.put(fileReference, 0.0); - } long end = System.currentTimeMillis() + timeout.toMillis(); boolean downloadStarted = false; while ((System.currentTimeMillis() < end) && !downloadStarted) { @@ -77,7 +73,12 @@ public class FileReferenceDownloader { } void addToDownloadQueue(FileReferenceDownload fileReferenceDownload) { - log.log(LogLevel.DEBUG, "Will download file reference '" + fileReferenceDownload.fileReference().value() + "' with timeout " + downloadTimeout); + FileReference fileReference = fileReferenceDownload.fileReference(); + log.log(LogLevel.DEBUG, "Will download file reference '" + fileReference.value() + "' with timeout " + downloadTimeout); + synchronized (downloads) { + downloads.put(fileReference, fileReferenceDownload); + downloadStatus.put(fileReference, 0.0); + } downloadExecutor.submit(() -> startDownload(downloadTimeout, fileReferenceDownload)); } @@ -118,7 +119,7 @@ public class FileReferenceDownloader { log.log(LogLevel.WARNING, "Request failed. Req: " + request + "\nSpec: " + connection.getAddress() + ", error code: " + request.errorCode()); if (request.isError() && request.errorCode() == ErrorCode.CONNECTION || request.errorCode() == ErrorCode.TIMEOUT) { - log.log(LogLevel.WARNING, "Setting error for connection " + connection.getAddress()); + log.log(LogLevel.INFO, "Mark connection " + connection.getAddress() + " with error"); connectionPool.setError(connection, request.errorCode()); } return false; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java index b01bd96e506..e60ba141c9b 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class ForEachExpression extends CompositeExpression { diff --git a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java index 26abdd43815..65fa83598b6 100644 --- a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java +++ b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java @@ -3,6 +3,9 @@ package com.yahoo.logserver; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; import com.yahoo.io.SelectLoopHook; @@ -20,18 +23,14 @@ import com.yahoo.logserver.handlers.LogHandler; public class LogDispatcher implements LogHandler, SelectLoopHook { private static final Logger log = Logger.getLogger(LogDispatcher.class.getName()); - private final List<LogHandler> handlers = new ArrayList<>(); - private int messageCount = 0; - private boolean hasBeenShutDown = false; - private boolean batchedMode = false; + private final List<LogHandler> handlers = new CopyOnWriteArrayList<>(); + private final AtomicInteger messageCount = new AtomicInteger(0); + private final AtomicBoolean batchedMode = new AtomicBoolean(false); private final int batchSize = 5000; - private List<LogMessage> currentBatchList; - private int roundCount = 0; - @SuppressWarnings("unused") - private int lastRoundCount = 0; + private final AtomicBoolean hasBeenShutDown = new AtomicBoolean(false); + private List<LogMessage> currentBatchList = null; - public LogDispatcher() { - } + public LogDispatcher() { } /** * Dispatches a message to all the LogHandler instances we've @@ -41,47 +40,53 @@ public class LogDispatcher implements LogHandler, SelectLoopHook { * @param msg The LogMessage instance we wish to dispatch to the * plugins */ - public synchronized void handle(LogMessage msg) { + public void handle(LogMessage msg) { if (msg == null) { throw new NullPointerException("LogMessage was null"); } - if (batchedMode) { + if (batchedMode.get()) { addToBatch(msg); } else { - for (LogHandler h : handlers) { - h.handle(msg); - } + send(msg); } - messageCount++; + messageCount.incrementAndGet(); } private void addToBatch(LogMessage msg) { - if (currentBatchList == null) { - currentBatchList = new ArrayList<LogMessage>(batchSize); - currentBatchList.add(msg); - return; - } + List<LogMessage> toSend = null; + synchronized (this) { + if (currentBatchList == null) { + currentBatchList = new ArrayList<LogMessage>(batchSize); + currentBatchList.add(msg); + return; + } - currentBatchList.add(msg); + currentBatchList.add(msg); - if (currentBatchList.size() == batchSize) { - flushBatch(); + if (currentBatchList.size() == batchSize) { + toSend = stealBatch(); + } } + flushBatch(toSend); } - private void flushBatch() { - List<LogMessage> todo; - synchronized(this) { - todo = currentBatchList; - currentBatchList = null; + private void send(List<LogMessage> messages) { + for (LogHandler ht : handlers) { + ht.handle(messages); } - if (todo == null) return; + } + private void send(LogMessage message) { for (LogHandler ht : handlers) { - ht.handle(todo); + ht.handle(message); } } + private void flushBatch(List<LogMessage> todo) { + if (todo == null) { return; } + send(todo); + } + public void handle(List<LogMessage> messages) { throw new IllegalStateException("method not supported"); } @@ -94,12 +99,20 @@ public class LogDispatcher implements LogHandler, SelectLoopHook { * but lists of same. */ public void setBatchedMode(boolean batchedMode) { - this.batchedMode = batchedMode; + this.batchedMode.set(batchedMode); } - public synchronized void flush() { - if (batchedMode) { - flushBatch(); + private List<LogMessage> stealBatch() { + List<LogMessage> toSend = null; + synchronized (this) { + toSend = currentBatchList; + currentBatchList = null; + } + return toSend; + } + public void flush() { + if (batchedMode.get()) { + flushBatch(stealBatch()); } for (LogHandler h : handlers) { @@ -110,15 +123,15 @@ public class LogDispatcher implements LogHandler, SelectLoopHook { } } - public synchronized void close() { - if (hasBeenShutDown) { + public void close() { + if (hasBeenShutDown.getAndSet(true)) { throw new IllegalStateException("Shutdown already in progress"); } - hasBeenShutDown = true; for (LogHandler ht : handlers) { if (ht instanceof Thread) { log.fine("Stopping " + ht); + // Todo: Very bad, never do.... ((Thread) ht).interrupt(); } } @@ -134,17 +147,18 @@ public class LogDispatcher implements LogHandler, SelectLoopHook { * <p> * If the thread is not alive it will be start()'ed. */ - public synchronized void registerLogHandler(LogHandler ht) { - if (hasBeenShutDown) { - throw new IllegalStateException("Tried to register LogHandler on" + - " LogDispatcher which was shut down"); + public void registerLogHandler(LogHandler ht) { + if (hasBeenShutDown.get()) { + throw new IllegalStateException("Tried to register LogHandler on LogDispatcher which was shut down"); } - if (handlers.contains(ht)) { - log.warning("LogHandler was already registered: " + ht); - return; + synchronized (this) { + if (handlers.contains(ht)) { + log.warning("LogHandler was already registered: " + ht); + return; + } + handlers.add(ht); } - handlers.add(ht); if ((ht instanceof Thread) && (! ((Thread) ht).isAlive())) { ((Thread) ht).start(); @@ -166,19 +180,16 @@ public class LogDispatcher implements LogHandler, SelectLoopHook { * * @return Returns the number of messages that we have seen. */ - public synchronized int getMessageCount() { - return messageCount; + public int getMessageCount() { + return messageCount.get(); } /** * Hook which is called when the select loop has finished. */ public void selectLoopHook(boolean before) { - if (batchedMode) { - flushBatch(); + if (batchedMode.get()) { + flushBatch(stealBatch()); } - - lastRoundCount = messageCount - roundCount; - roundCount = messageCount; } } diff --git a/node-admin/pom.xml b/node-admin/pom.xml index 9484525a8ec..3bd7eb04e8b 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -157,6 +157,23 @@ </compilerArgs> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <outputDirectory>target/node-admin-app/components</outputDirectory> + <includeArtifactIds>bcprov-jdk15on,bcpkix-jdk15on</includeArtifactIds> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index 99733552f76..f2f9e46c5b8 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -11,7 +11,9 @@ <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> - <nodes type="host"/> + <config name="vespa.hosted.dockerapi.docker"> + <uri>unix:///var/run/docker.sock</uri> + </config> <preprocess:include file="variant.xml" required="false"/> </jdisc> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java index 15ade142b5d..a678b8607fd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; +package com.yahoo.vespa.hosted.node.admin.component; import com.yahoo.concurrent.classlock.ClassLocking; import com.yahoo.net.HostName; @@ -7,11 +7,13 @@ import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; -import com.yahoo.vespa.hosted.node.admin.component.AdminComponent; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; @@ -20,7 +22,6 @@ import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; -import com.yahoo.vespa.hosted.node.admin.util.Environment; import java.time.Clock; import java.time.Duration; @@ -35,24 +36,37 @@ public class DockerAdminComponent implements AdminComponent { private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30); private final ConfigServerConfig configServerConfig; - private final NodeAdminConfig config; private final Docker docker; private final MetricReceiverWrapper metricReceiver; - private final ClassLocking classLocking; - - private ConfigServerHttpRequestExecutor requestExecutor; + private final Optional<ClassLocking> classLocking; + private Optional<Environment> environment = Optional.empty(); + private Optional<ConfigServerHttpRequestExecutor> requestExecutor = Optional.empty(); private Optional<NodeAdminStateUpdaterImpl> nodeAdminStateUpdater = Optional.empty(); public DockerAdminComponent(ConfigServerConfig configServerConfig, - NodeAdminConfig config, Docker docker, MetricReceiverWrapper metricReceiver, ClassLocking classLocking) { + this(configServerConfig, docker, metricReceiver, Optional.empty(), Optional.of(classLocking)); + } + + public DockerAdminComponent(ConfigServerConfig configServerConfig, + Docker docker, + MetricReceiverWrapper metricReceiver, + Environment environment) { + this(configServerConfig, docker, metricReceiver, Optional.of(environment), Optional.empty()); + } + + private DockerAdminComponent(ConfigServerConfig configServerConfig, + Docker docker, + MetricReceiverWrapper metricReceiver, + Optional<Environment> environment, + Optional<ClassLocking> classLocking) { this.configServerConfig = configServerConfig; - this.config = config; this.docker = docker; this.metricReceiver = metricReceiver; + this.environment = environment; this.classLocking = classLocking; } @@ -62,15 +76,25 @@ public class DockerAdminComponent implements AdminComponent { return; } - Environment environment = new Environment(configServerConfig); - requestExecutor = ConfigServerHttpRequestExecutor.create( - environment.getConfigServerUris(), - environment.getKeyStoreOptions(), - environment.getTrustStoreOptions(), - environment.getAthenzIdentity()); + nodeAdminStateUpdater = Optional.of(createNodeAdminStateUpdater()); + nodeAdminStateUpdater.get().start(); + } + + private NodeAdminStateUpdaterImpl createNodeAdminStateUpdater() { + if (!environment.isPresent()) { + environment = Optional.of(new Environment(configServerConfig)); + } - NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor); - Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); + if (!requestExecutor.isPresent()) { + requestExecutor = Optional.of(ConfigServerHttpRequestExecutor.create( + environment.get().getConfigServerUris(), + environment.get().getKeyStoreOptions(), + environment.get().getTrustStoreOptions(), + environment.get().getAthenzIdentity())); + } + + NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor.get()); + Orchestrator orchestrator = new OrchestratorImpl(requestExecutor.get()); Clock clock = Clock.systemUTC(); String dockerHostHostName = HostName.getLocalhost(); @@ -79,14 +103,14 @@ public class DockerAdminComponent implements AdminComponent { docker.start(); DockerOperations dockerOperations = new DockerOperationsImpl( docker, - environment, + environment.get(), processExecuter); StorageMaintainer storageMaintainer = new StorageMaintainer( dockerOperations, processExecuter, metricReceiver, - environment, + environment.get(), clock); AclMaintainer aclMaintainer = new AclMaintainer( @@ -101,7 +125,7 @@ public class DockerAdminComponent implements AdminComponent { dockerOperations, storageMaintainer, aclMaintainer, - environment, + environment.get(), clock, NODE_AGENT_SCAN_INTERVAL); @@ -113,7 +137,7 @@ public class DockerAdminComponent implements AdminComponent { metricReceiver, clock); - nodeAdminStateUpdater = Optional.of(new NodeAdminStateUpdaterImpl( + return new NodeAdminStateUpdaterImpl( nodeRepository, orchestrator, storageMaintainer, @@ -121,9 +145,7 @@ public class DockerAdminComponent implements AdminComponent { dockerHostHostName, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, - classLocking)); - - nodeAdminStateUpdater.get().start(); + classLocking); } @Override @@ -132,10 +154,9 @@ public class DockerAdminComponent implements AdminComponent { return; } - nodeAdminStateUpdater.get().stop(); - requestExecutor.close(); + nodeAdminStateUpdater.ifPresent(NodeAdminStateUpdaterImpl::stop); + requestExecutor.ifPresent(ConfigServerHttpRequestExecutor::close); nodeAdminStateUpdater = Optional.empty(); - // TODO: Also stop docker } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java index 0415bbc34c2..7376a59bd5c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.util; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.component; import com.google.common.base.Strings; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -7,6 +7,8 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; +import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; +import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; import java.net.InetAddress; import java.net.URI; @@ -46,7 +48,7 @@ public class Environment { private final InetAddressResolver inetAddressResolver; private final PathResolver pathResolver; private final List<String> logstashNodes; - private final String feedEndpoint; + private final Optional<String> feedEndpoint; private final Optional<KeyStoreOptions> keyStoreOptions; private final Optional<KeyStoreOptions> trustStoreOptions; private final Optional<AthenzIdentity> athenzIdentity; @@ -56,30 +58,42 @@ public class Environment { } public Environment(ConfigServerConfig configServerConfig) { + this(configServerConfig, + getEnvironmentVariable(ENVIRONMENT), + getEnvironmentVariable(REGION), + new PathResolver(), + Optional.of(getEnvironmentVariable(COREDUMP_FEED_ENDPOINT))); + } + + public Environment(ConfigServerConfig configServerConfig, + String hostedEnvironment, + String hostedRegion, + PathResolver pathResolver, + Optional<String> coreDumpFeedEndpoint) { this(createConfigServerUris( configServerConfig.scheme(), configServerConfig.hosts(), configServerConfig.port()), - getEnvironmentVariable(ENVIRONMENT), - getEnvironmentVariable(REGION), - Defaults.getDefaults().vespaHostname(), - new InetAddressResolver(), - new PathResolver(), - getLogstashNodesFromEnvironment(), - getEnvironmentVariable(COREDUMP_FEED_ENDPOINT), - - createKeyStoreOptions( - configServerConfig.keyStoreConfig().path(), - configServerConfig.keyStoreConfig().password().toCharArray(), - configServerConfig.keyStoreConfig().type().name()), - createKeyStoreOptions( - configServerConfig.trustStoreConfig().path(), - configServerConfig.trustStoreConfig().password().toCharArray(), - configServerConfig.trustStoreConfig().type().name()), - createAthenzIdentity( - configServerConfig.athenzDomain(), - configServerConfig.serviceName()) + hostedEnvironment, + hostedRegion, + Defaults.getDefaults().vespaHostname(), + new InetAddressResolver(), + pathResolver, + getLogstashNodesFromEnvironment(), + coreDumpFeedEndpoint, + + createKeyStoreOptions( + configServerConfig.keyStoreConfig().path(), + configServerConfig.keyStoreConfig().password().toCharArray(), + configServerConfig.keyStoreConfig().type().name()), + createKeyStoreOptions( + configServerConfig.trustStoreConfig().path(), + configServerConfig.trustStoreConfig().password().toCharArray(), + configServerConfig.trustStoreConfig().type().name()), + createAthenzIdentity( + configServerConfig.athenzDomain(), + configServerConfig.serviceName()) ); } @@ -90,7 +104,7 @@ public class Environment { InetAddressResolver inetAddressResolver, PathResolver pathResolver, List<String> logstashNodes, - String feedEndpoint, + Optional<String> feedEndpoint, Optional<KeyStoreOptions> keyStoreOptions, Optional<KeyStoreOptions> trustStoreOptions, Optional<AthenzIdentity> athenzIdentity) { @@ -166,7 +180,7 @@ public class Environment { return pathResolver; } - public String getCoredumpFeedEndpoint() { + public Optional<String> getCoredumpFeedEndpoint() { return feedEndpoint; } @@ -245,7 +259,7 @@ public class Environment { private InetAddressResolver inetAddressResolver; private PathResolver pathResolver; private List<String> logstashNodes = Collections.emptyList(); - private String feedEndpoint; + private Optional<String> feedEndpoint = Optional.empty(); private KeyStoreOptions keyStoreOptions; private KeyStoreOptions trustStoreOptions; private AthenzIdentity athenzIdentity; @@ -288,7 +302,7 @@ public class Environment { } public Builder feedEndpoint(String feedEndpoint) { - this.feedEndpoint = feedEndpoint; + this.feedEndpoint = Optional.of(feedEndpoint); return this; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java index b6b64dbf5dd..d2c09aae22a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java @@ -3,19 +3,39 @@ package com.yahoo.vespa.hosted.node.admin.component; /** * This class is thread unsafe: All method calls MUST be exclusive and serialized. + * + * In a specialized environment it is possible to provide a richer context than TaskContext: + * - Define a subclass T of TaskContext with the additional functionality. + * - Define task classes that implement IdempotentTask<T>. */ -public interface IdempotentTask { - String name(); +public interface IdempotentTask<T extends TaskContext> { + /** + * A short id of the task to e.g. identify the task in the log. + * + * Prefer PascalCase and without white-space. + * + * Example: "EnableDocker" + */ + default String name() { return getClass().getSimpleName(); } /** - * Execute an administrative task to converge the system towards some ideal state. + * Execute an administrative task to converge towards some ideal state, whether it is + * system state or in-memory Java state. * * converge() must be idempotent: it may be called any number of times, or - * interrupted at any time e.g. by `kill -9`. The caller must ensure there is at - * most one invocation of converge() on this instance at any given time. + * interrupted at any time e.g. by `kill -9`. + * + * converge() is not thread safe: The caller must ensure there is at most one invocation + * of converge() at any given time. * - * @return false if the system was already converged, i.e. converge() was a no-op. + * @return false if already converged, i.e. was a no-op. A typical sequence of converge() + * calls on a IdempotentTask will consist of: + * - Any number of calls that throws an exception due to some issues. Assuming + * no exceptions were thrown, or the issue eventually resolved itself... + * (convergence failure) + * - Returns true once (converged just now) + * - Returns false for all further calls (already converged) * @throws RuntimeException (or a subclass) if the task is unable to converge. */ - boolean converge(TaskContext context); + boolean converge(T context); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/PathResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java index 2e26ff0773b..f70f451d33d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/PathResolver.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.util; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.component; import java.nio.file.Path; import java.nio.file.Paths; @@ -8,9 +8,9 @@ import java.nio.file.Paths; * @author freva */ public class PathResolver { - static final Path ROOT = Paths.get("/"); + public static final Path ROOT = Paths.get("/"); + public static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage"); - private static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage"); private final Path applicationStoragePathForNodeAdmin; private final Path applicationStoragePathForHost; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskComponent.java index c54f9ee00c8..cbe9b32cc47 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskComponent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskComponent.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.component; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.ChainedComponent; -public abstract class TaskComponent extends ChainedComponent implements IdempotentTask { +public abstract class TaskComponent extends ChainedComponent implements IdempotentTask<TaskContext> { protected TaskComponent(ComponentId id) { super(id); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java index d0a5570b8dc..0c49e478d6a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java @@ -50,10 +50,4 @@ public interface TaskContext { * or later if the task failed. Either way, it will only be called once. */ default void logOnFailure(Logger logger, Supplier<String> messageSupplier) {} - - /** - * Execute a task as a child of this task, and with its own sub-TaskContext. Please avoid - * excessive task hierarchies. - */ - boolean executeSubtask(IdempotentTask task); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java index 108d42114f7..bc0ec2c8700 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.component; import java.util.ArrayList; import java.util.List; -import java.util.function.Supplier; import java.util.logging.Logger; public class TestTaskContext implements TaskContext { @@ -18,9 +17,6 @@ public class TestTaskContext implements TaskContext { @Override public void log(Logger logger, String message) { } - @Override - public void logOnFailure(Logger logger, Supplier<String> messageSupplier) { } - public List<String> getSystemModificationLog() { return systemModifications; } @@ -28,9 +24,4 @@ public class TestTaskContext implements TaskContext { public void clearSystemModificationLog() { systemModifications.clear(); } - - @Override - public boolean executeSubtask(IdempotentTask task) { - throw new UnsupportedOperationException(); - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 88f5c9acfed..b30cac2476e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -12,8 +12,8 @@ import com.yahoo.vespa.hosted.dockerapi.DockerImpl; import com.yahoo.vespa.hosted.dockerapi.DockerNetworkCreator; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.iptables.NATCommand; -import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import java.io.IOException; @@ -45,6 +45,9 @@ public class DockerOperationsImpl implements DockerOperations { private static final String MANAGER_NAME = "node-admin"; + private static final String LOCAL_IPV6_PREFIX = "fd00::"; + private static final String DOCKER_CUSTOM_BRIDGE_NETWORK_NAME = "vespa-bridge"; + // Map of directories to mount and whether they should be writable by everyone private static final Map<String, Boolean> DIRECTORIES_TO_MOUNT = new HashMap<>(); @@ -63,7 +66,7 @@ public class DockerOperationsImpl implements DockerOperations { DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/yms_agent"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/ysar"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/ystatus"), false); - DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/zpe_policy_updater"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/zpu"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/cache"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/crash"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/db/jdisc"), false); @@ -80,6 +83,7 @@ public class DockerOperationsImpl implements DockerOperations { DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/yca"), true); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/ycore++"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/zookeeper"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/zpe"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("tmp"), false); } @@ -119,11 +123,16 @@ public class DockerOperationsImpl implements DockerOperations { .withAddCapability("SYS_PTRACE") // Needed for gcore, pstack etc. .withAddCapability("SYS_ADMIN"); // Needed for perf - if (!docker.networkNATed()) { - logger.info("Network not nated - setting up with specific ip address on a macvlan"); + if (!docker.networkNPTed()) { command.withIpAddress(nodeInetAddress); command.withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME); command.withVolume("/etc/hosts", "/etc/hosts"); // TODO This is probably not nessesary - review later + } else { + command.withIpAddress(NetworkPrefixTranslator.translate( + nodeInetAddress, + InetAddress.getByName(LOCAL_IPV6_PREFIX), + 64)); + command.withNetworkMode(DOCKER_CUSTOM_BRIDGE_NETWORK_NAME); } for (String pathInNode : DIRECTORIES_TO_MOUNT.keySet()) { @@ -143,17 +152,14 @@ public class DockerOperationsImpl implements DockerOperations { command.create(); if (isIPv6) { - if (!docker.networkNATed()) { + if (!docker.networkNPTed()) { docker.connectContainerToNetwork(containerName, "bridge"); } docker.startContainer(containerName); - setupContainerNetworkConnectivity(containerName, nodeInetAddress); + setupContainerNetworkConnectivity(containerName); } else { docker.startContainer(containerName); - if (docker.networkNATed()) { - setupContainerNetworkConnectivity(containerName, nodeInetAddress); - } } DIRECTORIES_TO_MOUNT.entrySet().stream().filter(Map.Entry::getValue).forEach(entry -> @@ -175,7 +181,7 @@ public class DockerOperationsImpl implements DockerOperations { logger.info("Deleting container " + containerName.asString()); docker.deleteContainer(containerName); - if (docker.networkNATed()) { + if (docker.networkNPTed()) { logger.info("Delete iptables NAT rules for " + containerName.asString()); try { InetAddress nodeInetAddress = environment.getInetAddressForHost(nodeSpec.hostname); @@ -221,16 +227,12 @@ public class DockerOperationsImpl implements DockerOperations { /** * For macvlan: + * <p> * Due to a bug in docker (https://github.com/docker/libnetwork/issues/1443), we need to manually set * IPv6 gateway in containers connected to more than one docker network - * - * For nat: - * Setup iptables NAT rules to map the hosts public ips to the containers */ - private void setupContainerNetworkConnectivity(ContainerName containerName, InetAddress externalAddress) throws IOException { - if (docker.networkNATed()) { - insertNAT(containerName, externalAddress); - } else { + private void setupContainerNetworkConnectivity(ContainerName containerName) throws IOException { + if (!docker.networkNPTed()) { InetAddress hostDefaultGateway = DockerNetworkCreator.getDefaultGatewayLinux(true); executeCommandInNetworkNamespace(containerName, "route", "-A", "inet6", "add", "default", "gw", hostDefaultGateway.getHostAddress(), "dev", "eth1"); @@ -274,7 +276,7 @@ public class DockerOperationsImpl implements DockerOperations { final String[] wrappedCommand = Stream.concat( Stream.of("sudo", "nsenter", String.format("--net=/host/proc/%d/ns/net", containerPid), "--"), Stream.of(command)) - .toArray(String[]::new); + .toArray(String[]::new); try { Pair<Integer, String> result = processExecuter.exec(wrappedCommand); @@ -326,25 +328,4 @@ public class DockerOperationsImpl implements DockerOperations { public void deleteUnusedDockerImages() { docker.deleteUnusedDockerImages(); } - - /** - * Only insert NAT rules if they don't exist (or else they compounded) - */ - private void insertNAT(ContainerName containerName, InetAddress externalAddress) throws IOException { - PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName); - String ipv6Str = docker.getGlobalIPv6Address(containerName); - - // Check if exist - String checkCommand = NATCommand.check(externalAddress, InetAddress.getByName(ipv6Str)); - Pair<Integer, String> result = processExecuter.exec(checkCommand); - if (result.getFirst() == 0 ) return; - - // Setup NAT - String natCommand = NATCommand.insert(externalAddress, InetAddress.getByName(ipv6Str)); - logger.info("Setting up NAT rules: " + natCommand); - result = processExecuter.exec(checkCommand); - if (result.getFirst() != 0 ) { - throw new IOException("Unable to setup NAT rule - error message: " + result.getSecond()); - } - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslator.java new file mode 100644 index 00000000000..a52dedb90e5 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslator.java @@ -0,0 +1,38 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * @author smorgrav + */ +package com.yahoo.vespa.hosted.node.admin.docker; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.UnknownHostException; + +class NetworkPrefixTranslator { + + /** + * For NPTed networks we want to find the private address from a public. + * + * @param address The original address to translate + * @param prefix The prefix address + * @param subnetSize in bits - e.g a /64 subnet equals 64 bits + * @return The translated address + */ + static Inet6Address translate(InetAddress address, InetAddress prefix, int subnetSize) { + + byte[] originalAddress = address.getAddress(); + byte[] prefixAddress = prefix.getAddress(); + byte[] translatedAddress = new byte[16]; + + for (int i = 0; i < 16; i++) { + translatedAddress[i] = i < subnetSize / 8 ? prefixAddress[i] : originalAddress[i]; + } + + try { + return (Inet6Address) InetAddress.getByAddress(address.getHostName(), translatedAddress); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java index dde2a39da67..422e47fe83a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.logging; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import java.util.Optional; import java.util.stream.Collectors; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 257247bf44a..c12f1168240 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import com.yahoo.vespa.hosted.node.admin.util.SecretAgentScheduleMaker; @@ -223,6 +223,11 @@ public class StorageMaintainer { } private void addHandleCoredumpsCommand(MaintainerExecutor maintainerExecutor, ContainerName containerName, ContainerNodeSpec nodeSpec) { + if (!environment.getCoredumpFeedEndpoint().isPresent()) { + // Core dump handling is disabled. + return; + } + Map<String, Object> attributes = new HashMap<>(); attributes.put("hostname", nodeSpec.hostname); attributes.put("parent_hostname", HostName.getLocalhost()); @@ -243,7 +248,7 @@ public class StorageMaintainer { .withArgument("doneCoredumpsPath", environment.pathInNodeAdminToDoneCoredumps()) .withArgument("coredumpsPath", environment.pathInNodeAdminFromPathInNode( containerName, getDefaults().underVespaHome("var/crash"))) - .withArgument("feedEndpoint", environment.getCoredumpFeedEndpoint()) + .withArgument("feedEndpoint", environment.getCoredumpFeedEndpoint().get()) .withArgument("attributes", attributes); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java index b0631fc712b..95eb9e150a2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.component.AdminComponent; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; +import com.yahoo.vespa.hosted.node.admin.component.DockerAdminComponent; import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; import java.io.File; @@ -59,7 +60,7 @@ public class NodeAdminMain implements AutoCloseable { private AdminComponent selectAdminComponent(NodeAdminConfig config) { if (config.mainComponent == null) { - return new DockerAdminComponent(configServerConfig, config, docker, metricReceiver, classLocking); + return new DockerAdminComponent(configServerConfig, docker, metricReceiver, classLocking); } logger.log(LogLevel.INFO, () -> { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java index e4e66b57186..cdc7c9badaf 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java @@ -64,8 +64,8 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { private final String dockerHostHostName; private final Duration nodeAdminConvergeStateInterval; - private final ClassLocking classLocking; - private Optional<ClassLock> classLock; + private final Optional<ClassLocking> classLocking; + private Optional<ClassLock> classLock = Optional.empty(); private Instant lastTick; public NodeAdminStateUpdaterImpl( @@ -76,7 +76,7 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { String dockerHostHostName, Clock clock, Duration nodeAdminConvergeStateInterval, - ClassLocking classLocking) { + Optional<ClassLocking> classLocking) { log.info(objectToString() + ": Creating object"); this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; @@ -88,12 +88,14 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { this.lastTick = clock.instant(); this.loopThread = new Thread(() -> { - log.info(objectToString() + ": Acquiring lock"); - try { - classLock = Optional.of(classLocking.lockWhile(NodeAdminStateUpdater.class, () -> !terminated.get())); - } catch (LockInterruptException e) { - classLock = Optional.empty(); - return; + if (classLocking.isPresent()) { + log.info(objectToString() + ": Acquiring lock"); + try { + classLock = Optional.of(classLocking.get().lockWhile(NodeAdminStateUpdater.class, () -> !terminated.get())); + } catch (LockInterruptException e) { + classLock = Optional.empty(); + return; + } } log.info(objectToString() + ": Starting threads and schedulers"); @@ -308,7 +310,7 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { throw new RuntimeException("Can not re-stop a node agent."); } - classLocking.interrupt(); + classLocking.ifPresent(ClassLocking::interrupt); // First we need to stop NodeAdminStateUpdaterImpl thread to make sure no new NodeAgents are spawned signalWorkToBeDone(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index edf4f059fc2..f05a4054924 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -20,7 +20,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import com.yahoo.vespa.hosted.provision.Node; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java index c294d9ee806..073978530a5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java @@ -2,12 +2,9 @@ package com.yahoo.vespa.hosted.node.admin.task.util.systemd; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcess; -import com.yahoo.vespa.hosted.node.admin.task.util.process.Command; -import com.yahoo.vespa.hosted.node.admin.task.util.process.UnexpectedOutputException; +import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; import java.util.Objects; -import java.util.function.Function; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -27,7 +24,7 @@ public class SystemCtl { private static final Pattern UNIT_FILE_STATE_PROPERTY_PATTERN = createPropertyPattern("UnitFileState"); private static final Pattern ACTIVE_STATE_PROPERTY_PATTERN = createPropertyPattern("ActiveState"); - private final Function<TaskContext, Command> commandSupplier; + private final Terminal terminal; private static Pattern createPropertyPattern(String propertyName) { if (!PROPERTY_NAME_PATTERN.matcher(propertyName).matches()) { @@ -40,8 +37,8 @@ public class SystemCtl { return Pattern.compile(regex); } - public SystemCtl() { - this(Command::new); + public SystemCtl(Terminal terminal) { + this.terminal = terminal; } public SystemCtlEnable enable(String unit) { return new SystemCtlEnable(unit); } @@ -56,8 +53,7 @@ public class SystemCtl { } protected boolean isAlreadyConverged(TaskContext context) { - ChildProcess showProcess = systemCtlShow(context); - String unitFileState = extractProperty(showProcess, UNIT_FILE_STATE_PROPERTY_PATTERN); + String unitFileState = getSystemCtlProperty(context, UNIT_FILE_STATE_PROPERTY_PATTERN); return Objects.equals(unitFileState, "enabled"); } } @@ -68,8 +64,7 @@ public class SystemCtl { } protected boolean isAlreadyConverged(TaskContext context) { - ChildProcess showProcess = systemCtlShow(context); - String unitFileState = extractProperty(showProcess, UNIT_FILE_STATE_PROPERTY_PATTERN); + String unitFileState = getSystemCtlProperty(context, UNIT_FILE_STATE_PROPERTY_PATTERN); return Objects.equals(unitFileState, "disabled"); } } @@ -80,8 +75,7 @@ public class SystemCtl { } protected boolean isAlreadyConverged(TaskContext context) { - ChildProcess showProcess = systemCtlShow(context); - String activeState = extractProperty(showProcess, ACTIVE_STATE_PROPERTY_PATTERN); + String activeState = getSystemCtlProperty(context, ACTIVE_STATE_PROPERTY_PATTERN); return Objects.equals(activeState, "active"); } } @@ -92,8 +86,7 @@ public class SystemCtl { } protected boolean isAlreadyConverged(TaskContext context) { - ChildProcess showProcess = systemCtlShow(context); - String activeState = extractProperty(showProcess, ACTIVE_STATE_PROPERTY_PATTERN); + String activeState = getSystemCtlProperty(context, ACTIVE_STATE_PROPERTY_PATTERN); return Objects.equals(activeState, "inactive"); } } @@ -124,48 +117,40 @@ public class SystemCtl { return false; } - commandSupplier.apply(context) + terminal.newCommandLine(context) .add("systemctl", command, unit) - .spawn(logger) - .waitForTermination() - .throwIfFailed(); + .execute(); return true; } /** - * Find the systemd property value of the property (given by propertyPattern) - * matching the 'systemctl show' output (given by showProcess). + * @param propertyPattern Pattern to match the output of systemctl show command with + * exactly 1 group. The matchng group must exist. + * @return The matched group from the 'systemctl show' output. */ - protected String extractProperty(ChildProcess showProcess, Pattern propertyPattern) { - String output = showProcess.getUtf8Output(); - Matcher matcher = propertyPattern.matcher(output); - if (!matcher.find()) { - throw new UnexpectedOutputException( - "Output does not match '" + propertyPattern + "'", showProcess); - } else if (matcher.groupCount() != 1) { - throw new IllegalArgumentException("Property pattern must have exactly 1 group"); - } - - return matcher.group(1); - } - - protected ChildProcess systemCtlShow(TaskContext context) { - ChildProcess process = commandSupplier.apply(context) + protected String getSystemCtlProperty(TaskContext context, Pattern propertyPattern) { + return terminal.newCommandLine(context) .add("systemctl", "show", unit) - .spawnProgramWithoutSideEffects() - .waitForTermination() - .throwIfFailed(); + .executeSilently() + .mapOutput(output -> extractProperty(output, propertyPattern)); + } + } - // Make sure we're able to parse UTF-8 output. - process.getUtf8Output(); - return process; + /** + * Find the systemd property value of the property (given by propertyPattern) + * matching the 'systemctl show' output (given by showProcess). + */ + private static String extractProperty(String showOutput, Pattern propertyPattern) { + Matcher matcher = propertyPattern.matcher(showOutput); + if (!matcher.find()) { + throw new IllegalArgumentException("Pattern '" + propertyPattern + + "' didn't match output"); + } else if (matcher.groupCount() != 1) { + throw new IllegalArgumentException("Property pattern must have exactly 1 group"); } - } - // For testing - SystemCtl(Function<TaskContext, Command> commandSupplier) { - this.commandSupplier = commandSupplier; + return matcher.group(1); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java index dbb4c909a4b..26a88e39e14 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java @@ -2,13 +2,12 @@ package com.yahoo.vespa.hosted.node.admin.task.util.yum; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcess; -import com.yahoo.vespa.hosted.node.admin.task.util.process.Command; +import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; +import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.function.Supplier; import java.util.logging.Logger; import java.util.regex.Pattern; @@ -22,11 +21,11 @@ public class Yum { private static final Pattern REMOVE_NOOP_PATTERN = Pattern.compile("(?dm)^No Packages marked for removal$"); private final TaskContext taskContext; - private final Supplier<Command> commandSupplier; + private final Terminal terminal; - public Yum(TaskContext taskContext) { + public Yum(TaskContext taskContext, Terminal terminal) { this.taskContext = taskContext; - this.commandSupplier = () -> new Command(taskContext); + this.terminal = terminal; } /** @@ -49,7 +48,7 @@ public class Yum { Pattern noopPattern) { return new GenericYumCommand( taskContext, - commandSupplier, + terminal, yumCommand, Arrays.asList(packages), noopPattern); @@ -59,19 +58,19 @@ public class Yum { private static Logger logger = Logger.getLogger(GenericYumCommand.class.getName()); private final TaskContext taskContext; - private final Supplier<Command> commandSupplier; + private final Terminal terminal; private final String yumCommand; private final List<String> packages; private final Pattern commandOutputNoopPattern; private Optional<String> enabledRepo = Optional.empty(); private GenericYumCommand(TaskContext taskContext, - Supplier<Command> commandSupplier, + Terminal terminal, String yumCommand, List<String> packages, Pattern commandOutputNoopPattern) { this.taskContext = taskContext; - this.commandSupplier = commandSupplier; + this.terminal = terminal; this.yumCommand = yumCommand; this.packages = packages; this.commandOutputNoopPattern = commandOutputNoopPattern; @@ -88,30 +87,22 @@ public class Yum { } public boolean converge() { - Command command = commandSupplier.get(); - command.add("yum", yumCommand, "--assumeyes"); - enabledRepo.ifPresent(repo -> command.add("--enablerepo=" + repo)); - command.add(packages); - ChildProcess childProcess = command - .spawnProgramWithoutSideEffects() - .waitForTermination() - .throwIfFailed(); + CommandLine commandLine = terminal.newCommandLine(taskContext); + commandLine.add("yum", yumCommand, "--assumeyes"); + enabledRepo.ifPresent(repo -> commandLine.add("--enablerepo=" + repo)); + commandLine.add(packages); // There's no way to figure out whether a yum command would have been a no-op. // Therefore, run the command and parse the output to decide. - String output = childProcess.getUtf8Output(); - if (commandOutputNoopPattern.matcher(output).find()) { - return false; - } else { - childProcess.logAsModifyingSystemAfterAll(logger); - return true; + boolean modifiedSystem = commandLine + .executeSilently() + .mapOutput(output -> !commandOutputNoopPattern.matcher(output).find()); + + if (modifiedSystem) { + commandLine.recordSilentExecutionAsSystemModification(); } - } - } - // For testing - Yum(TaskContext taskContext, Supplier<Command> commandSupplier) { - this.taskContext = taskContext; - this.commandSupplier = commandSupplier; + return modifiedSystem; + } } } diff --git a/node-admin/src/main/sh/node-admin.sh b/node-admin/src/main/sh/node-admin.sh index 3196ff9fa32..b2dac920c56 100755 --- a/node-admin/src/main/sh/node-admin.sh +++ b/node-admin/src/main/sh/node-admin.sh @@ -67,6 +67,14 @@ EOF exit 1 } +Start() { + "$VESPA_HOME"/libexec/vespa/standalone-container.sh start -s node-admin -u root "$@" +} + +Stop() { + "$VESPA_HOME"/libexec/vespa/standalone-container.sh stop -s node-admin -u root "$@" +} + if (( $# == 0 )); then Usage fi @@ -75,11 +83,11 @@ command="$1" shift case "$command" in - start) - "$VESPA_HOME"/libexec/vespa/standalone-container.sh start -s node-admin -u root "$@" - ;; - stop) - "$VESPA_HOME"/libexec/vespa/standalone-container.sh stop -s node-admin -u root "$@" + start) Start "$@" ;; + stop) Stop "$@" ;; + restart) + Stop "$@" + Start "$@" ;; *) Usage ;; esac diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index 616a18d2f2f..034c9352a10 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import org.junit.Test; import org.mockito.InOrder; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslatorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslatorTest.java new file mode 100644 index 00000000000..96afe685a61 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslatorTest.java @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * @author smorgrav + */ +package com.yahoo.vespa.hosted.node.admin.docker; + +import org.junit.Assert; +import org.junit.Test; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.UnknownHostException; + +public class NetworkPrefixTranslatorTest { + + @Test + public void translator_with_valid_parameters() throws UnknownHostException { + + // Test simplest possible address + Inet6Address original = (Inet6Address)InetAddress.getByName("2001:db8::1"); + Inet6Address prefix = (Inet6Address)InetAddress.getByName("fd00::"); + Inet6Address translated = NetworkPrefixTranslator.translate(original, prefix, 64); + Assert.assertEquals("fd00:0:0:0:0:0:0:1", translated.getHostAddress()); + + + // Test an actual aws address we use + original = (Inet6Address)InetAddress.getByName("2600:1f16:f34:5300:ccc6:1703:b7c2:369d"); + translated = NetworkPrefixTranslator.translate(original, prefix, 64); + Assert.assertEquals("fd00:0:0:0:ccc6:1703:b7c2:369d", translated.getHostAddress()); + + // Test different subnet size + translated = NetworkPrefixTranslator.translate(original, prefix, 48); + Assert.assertEquals("fd00:0:0:5300:ccc6:1703:b7c2:369d", translated.getHostAddress()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 3cfc67824ed..d22cb968969 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -14,15 +14,16 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; -import com.yahoo.vespa.hosted.node.admin.util.PathResolver; +import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.file.Paths; import java.time.Clock; import java.time.Duration; +import java.util.Optional; import java.util.function.Function; import static org.mockito.Matchers.any; @@ -67,7 +68,8 @@ public class DockerTester implements AutoCloseable { orchestratorMock, dockerOperations, storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC()); nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, storageMaintainer, - nodeAdmin, "basehostname", clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, new ClassLocking()); + nodeAdmin, "basehostname", clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, + Optional.of(new ClassLocking())); nodeAdminStateUpdater.start(); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java index a0e122d99fc..e4b6558e8e1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java @@ -15,14 +15,14 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; -import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.provision.Node; import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; @@ -46,6 +46,7 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Collections; +import java.util.Optional; import java.util.function.Function; import java.util.logging.Logger; @@ -243,7 +244,8 @@ public class RunInContainerTest { storageMaintainer, aclMaintainer, environment, Clock.systemUTC(), NODE_AGENT_SCAN_INTERVAL); private final NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperationsMock, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC()); private final NodeAdminStateUpdaterImpl nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, - orchestratorMock, storageMaintainer, nodeAdmin, "localhost.test.yahoo.com", Clock.systemUTC(), NODE_ADMIN_CONVERGE_STATE_INTERVAL, new ClassLocking()); + orchestratorMock, storageMaintainer, nodeAdmin, "localhost.test.yahoo.com", + Clock.systemUTC(), NODE_ADMIN_CONVERGE_STATE_INTERVAL, Optional.of(new ClassLocking())); public NodeAdminProviderWithMocks() { nodeAdminStateUpdater.start(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java index 67627ee1a83..443948db450 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import java.time.Clock; import java.util.Optional; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java index 399579a7ce7..ad50041ab69 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.logging; import com.google.common.collect.ImmutableList; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 38dd11a7a51..6aeafebaea7 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -9,8 +9,8 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.util.Environment; -import com.yahoo.vespa.hosted.node.admin.util.PathResolver; +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Rule; import org.junit.Test; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java index 7920d0cad29..1ab24fe8f9a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -47,7 +48,8 @@ public class NodeAdminStateUpdaterImplTest { private final Duration convergeStateInterval = Duration.ofSeconds(30); private final NodeAdminStateUpdaterImpl refresher = spy(new NodeAdminStateUpdaterImpl( - nodeRepository, orchestrator, storageMaintainer, nodeAdmin, parentHostname, clock, convergeStateInterval, null)); + nodeRepository, orchestrator, storageMaintainer, nodeAdmin, parentHostname, clock, + convergeStateInterval, Optional.empty())); @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 8067bf9ba69..f1852fda8a3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -17,9 +17,9 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; -import com.yahoo.vespa.hosted.node.admin.util.PathResolver; +import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; import org.mockito.InOrder; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java index 53aafe6b47c..085402410b3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java @@ -2,13 +2,10 @@ package com.yahoo.vespa.hosted.node.admin.task.util.systemd; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.Command; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandException; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestCommandSupplier; +import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureException; +import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import org.junit.Test; -import java.util.function.Function; - import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -19,48 +16,45 @@ import static org.mockito.Mockito.mock; */ public class SystemCtlTest { private final TaskContext taskContext = mock(TaskContext.class); - private final TestCommandSupplier testCommandSupplier = new TestCommandSupplier(taskContext); - private final Function<TaskContext, Command> commandSupplier = context -> testCommandSupplier.get(); + private final TestTerminal terminal = new TestTerminal(); @Test public void enable() throws Exception { - testCommandSupplier - .expectCommand( - "systemctl show docker", + terminal.expectCommand( + "systemctl show docker 2>&1", 0, "a=b\n" + "UnitFileState=disabled\n" + "bar=zoo\n") - .expectCommand("systemctl enable docker", 0, ""); + .expectCommand("systemctl enable docker 2>&1", 0, ""); - SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(commandSupplier).enable("docker"); + SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(terminal).enable("docker"); assertTrue(enableDockerService.converge(taskContext)); } @Test public void enableIsNoop() throws Exception { - testCommandSupplier - .expectCommand( - "systemctl show docker", + terminal.expectCommand( + "systemctl show docker 2>&1", 0, "a=b\n" + "UnitFileState=enabled\n" + "bar=zoo\n") - .expectCommand("systemctl enable docker", 0, ""); + .expectCommand("systemctl enable docker 2>&1", 0, ""); - SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(commandSupplier).enable("docker"); + SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(terminal).enable("docker"); assertFalse(enableDockerService.converge(taskContext)); } @Test public void enableCommandFailre() throws Exception { - testCommandSupplier.expectCommand("systemctl show docker", 1, "error"); - SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(commandSupplier).enable("docker"); + terminal.expectCommand("systemctl show docker 2>&1", 1, "error"); + SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(terminal).enable("docker"); try { enableDockerService.converge(taskContext); fail(); - } catch (CommandException e) { + } catch (ChildProcessFailureException e) { // success } } @@ -68,43 +62,41 @@ public class SystemCtlTest { @Test public void start() throws Exception { - testCommandSupplier - .expectCommand( - "systemctl show docker", + terminal.expectCommand( + "systemctl show docker 2>&1", 0, "a=b\n" + "ActiveState=failed\n" + "bar=zoo\n") - .expectCommand("systemctl start docker", 0, ""); + .expectCommand("systemctl start docker 2>&1", 0, ""); - SystemCtl.SystemCtlStart startDockerService = new SystemCtl(commandSupplier).start("docker"); + SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); assertTrue(startDockerService.converge(taskContext)); } @Test public void startIsNoop() throws Exception { - testCommandSupplier - .expectCommand( - "systemctl show docker", + terminal.expectCommand( + "systemctl show docker 2>&1", 0, "a=b\n" + "ActiveState=active\n" + "bar=zoo\n") - .expectCommand("systemctl start docker", 0, ""); + .expectCommand("systemctl start docker 2>&1", 0, ""); - SystemCtl.SystemCtlStart startDockerService = new SystemCtl(commandSupplier).start("docker"); + SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); assertFalse(startDockerService.converge(taskContext)); } @Test public void startCommandFailre() throws Exception { - testCommandSupplier.expectCommand("systemctl show docker", 1, "error"); - SystemCtl.SystemCtlStart startDockerService = new SystemCtl(commandSupplier).start("docker"); + terminal.expectCommand("systemctl show docker 2>&1", 1, "error"); + SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); try { startDockerService.converge(taskContext); fail(); - } catch (CommandException e) { + } catch (ChildProcessFailureException e) { // success } } @@ -112,35 +104,33 @@ public class SystemCtlTest { @Test public void disable() throws Exception { - testCommandSupplier - .expectCommand( - "systemctl show docker", + terminal.expectCommand( + "systemctl show docker 2>&1", 0, "a=b\n" + "UnitFileState=enabled\n" + "bar=zoo\n") - .expectCommand("systemctl disable docker", 0, ""); + .expectCommand("systemctl disable docker 2>&1", 0, ""); - assertTrue(new SystemCtl(commandSupplier).disable("docker").converge(taskContext)); + assertTrue(new SystemCtl(terminal).disable("docker").converge(taskContext)); } @Test public void stop() throws Exception { - testCommandSupplier - .expectCommand( - "systemctl show docker", + terminal.expectCommand( + "systemctl show docker 2>&1", 0, "a=b\n" + "ActiveState=active\n" + "bar=zoo\n") - .expectCommand("systemctl stop docker", 0, ""); + .expectCommand("systemctl stop docker 2>&1", 0, ""); - assertTrue(new SystemCtl(commandSupplier).stop("docker").converge(taskContext)); + assertTrue(new SystemCtl(terminal).stop("docker").converge(taskContext)); } @Test public void restart() throws Exception { - testCommandSupplier.expectCommand("systemctl restart docker", 0, ""); - assertTrue(new SystemCtl(commandSupplier).restart("docker").converge(taskContext)); + terminal.expectCommand("systemctl restart docker 2>&1", 0, ""); + assertTrue(new SystemCtl(terminal).restart("docker").converge(taskContext)); } }
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java index c893c709950..f010ea07d99 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java @@ -2,8 +2,8 @@ package com.yahoo.vespa.hosted.node.admin.task.util.yum; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandException; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestCommandSupplier; +import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureException; +import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import org.junit.Before; import org.junit.Test; @@ -14,21 +14,21 @@ import static org.mockito.Mockito.mock; public class YumTest { TaskContext taskContext = mock(TaskContext.class); - TestCommandSupplier commandSupplier = new TestCommandSupplier(taskContext); + TestTerminal terminal = new TestTerminal(); @Before public void tearDown() { - commandSupplier.verifyInvocations(); + terminal.verifyAllCommandsExecuted(); } @Test public void testAlreadyInstalled() { - commandSupplier.expectCommand( - "yum install --assumeyes --enablerepo=repo-name package-1 package-2", + terminal.expectCommand( + "yum install --assumeyes --enablerepo=repo-name package-1 package-2 2>&1", 0, "foobar\nNothing to do\n"); - Yum yum = new Yum(taskContext, commandSupplier); + Yum yum = new Yum(taskContext, terminal); assertFalse(yum .install("package-1", "package-2") .enableRepo("repo-name") @@ -37,36 +37,36 @@ public class YumTest { @Test public void testAlreadyUpgraded() { - commandSupplier.expectCommand( - "yum upgrade --assumeyes package-1 package-2", + terminal.expectCommand( + "yum upgrade --assumeyes package-1 package-2 2>&1", 0, "foobar\nNo packages marked for update\n"); - assertFalse(new Yum(taskContext, commandSupplier) + assertFalse(new Yum(taskContext, terminal) .upgrade("package-1", "package-2") .converge()); } @Test public void testAlreadyRemoved() { - commandSupplier.expectCommand( - "yum remove --assumeyes package-1 package-2", + terminal.expectCommand( + "yum remove --assumeyes package-1 package-2 2>&1", 0, "foobar\nNo Packages marked for removal\n"); - assertFalse(new Yum(taskContext, commandSupplier) + assertFalse(new Yum(taskContext, terminal) .remove("package-1", "package-2") .converge()); } @Test public void testInstall() { - commandSupplier.expectCommand( - "yum install --assumeyes package-1 package-2", + terminal.expectCommand( + "yum install --assumeyes package-1 package-2 2>&1", 0, "installing, installing"); - Yum yum = new Yum(taskContext, commandSupplier); + Yum yum = new Yum(taskContext, terminal); assertTrue(yum .install("package-1", "package-2") .converge()); @@ -74,26 +74,26 @@ public class YumTest { @Test public void testInstallWithEnablerepo() { - commandSupplier.expectCommand( - "yum install --assumeyes --enablerepo=repo-name package-1 package-2", + terminal.expectCommand( + "yum install --assumeyes --enablerepo=repo-name package-1 package-2 2>&1", 0, "installing, installing"); - Yum yum = new Yum(taskContext, commandSupplier); + Yum yum = new Yum(taskContext, terminal); assertTrue(yum .install("package-1", "package-2") .enableRepo("repo-name") .converge()); } - @Test(expected = CommandException.class) + @Test(expected = ChildProcessFailureException.class) public void testFailedInstall() { - commandSupplier.expectCommand( - "yum install --assumeyes --enablerepo=repo-name package-1 package-2", + terminal.expectCommand( + "yum install --assumeyes --enablerepo=repo-name package-1 package-2 2>&1", 1, "error"); - Yum yum = new Yum(taskContext, commandSupplier); + Yum yum = new Yum(taskContext, terminal); yum.install("package-1", "package-2") .enableRepo("repo-name") .converge(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java index f5a661c8a66..49a10e34fa6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.node.admin.util; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/node-admin/vespa-node-admin.spec b/node-admin/vespa-node-admin.spec index 049e66bf22f..d270143976c 100644 --- a/node-admin/vespa-node-admin.spec +++ b/node-admin/vespa-node-admin.spec @@ -32,6 +32,7 @@ cp node-admin/src/main/application/services.xml "$app_dir" declare -a jar_components=( node-admin/target/node-admin-jar-with-dependencies.jar + node-admin/target/node-admin-app/components/* docker-api/target/docker-api-jar-with-dependencies.jar ) for path in "${jar_components[@]}"; do diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java index de08bdbe107..c82f4406015 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java @@ -127,7 +127,7 @@ public class CoreCollector { installStatePath.ifPresent(installState -> { try { - data.put("yinst_state", readInstallState(installState)); + data.put("install_state", readInstallState(installState)); } catch (Exception e) { logger.log(Level.WARNING, "Failed to read install state", e); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java index 1e95ca15c3d..6c8388c3f28 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java @@ -142,7 +142,7 @@ public class Maintainer { Path coredumpsPath = Paths.get(getFieldOrFail(arguments, "coredumpsPath").asString()); Path doneCoredumpsPath = Paths.get(getFieldOrFail(arguments, "doneCoredumpsPath").asString()); Map<String, Object> attributesMap = parseMap(arguments); - Optional<Path> installStatePath = SlimeUtils.optionalString(arguments.field("yinstStatePath")).map(Paths::get); + Optional<Path> installStatePath = SlimeUtils.optionalString(arguments.field("installStatePath")).map(Paths::get); String feedEndpoint = getFieldOrFail(arguments, "feedEndpoint").asString(); try { diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java index 869b53dc7a7..4b58c16bdc3 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java @@ -160,7 +160,7 @@ public class CoreCollectorTest { expectedData.put("bin_path", TEST_BIN_PATH.toString()); expectedData.put("backtrace", new ArrayList<>(GDB_BACKTRACE)); expectedData.put("backtrace_all_threads", new ArrayList<>(GDB_BACKTRACE)); - expectedData.put("yinst_state", new ArrayList<>(INSTALL_STATE)); + expectedData.put("install_state", new ArrayList<>(INSTALL_STATE)); expectedData.put("rpm_packages", new ArrayList<>(RPM_PACKAGES)); assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(INSTALL_STATE_PATH))); } @@ -171,7 +171,7 @@ public class CoreCollectorTest { mockExec(new String[]{"cat", INSTALL_STATE_PATH.toString()}, String.join("\n", INSTALL_STATE)); Map<String, Object> expectedData = new HashMap<>(); - expectedData.put("yinst_state", new ArrayList<>(INSTALL_STATE)); + expectedData.put("install_state", new ArrayList<>(INSTALL_STATE)); assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(INSTALL_STATE_PATH))); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 6a2376d748b..1d894f80eca 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; @@ -26,7 +27,7 @@ public abstract class ApplicationMaintainer extends Maintainer { private final Deployer deployer; - private final Executor deploymentExecutor = Executors.newCachedThreadPool(); + private final Executor deploymentExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory("node repo application maintainer")); protected ApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval, JobControl jobControl) { super(nodeRepository, interval, jobControl); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 6089cfe64c9..7726311ab2c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -248,11 +248,12 @@ public class NodeFailer extends Maintainer { // If the active node that we are trying to fail is of type host, we need to successfully fail all // the children nodes running on it before we fail the host boolean allTenantNodesFailedOutSuccessfully = true; + String reasonForChildFailure = "Failing due to parent host " + node.hostname() + " failure: " + reason; for (Node failingTenantNode : nodeRepository().getChildNodes(node.hostname())) { if (failingTenantNode.state() == Node.State.active) { - allTenantNodesFailedOutSuccessfully &= failActive(failingTenantNode, reason); + allTenantNodesFailedOutSuccessfully &= failActive(failingTenantNode, reasonForChildFailure); } else { - nodeRepository().fail(failingTenantNode.hostname(), Agent.system, reason); + nodeRepository().fail(failingTenantNode.hostname(), Agent.system, reasonForChildFailure); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index d9d7b4a5d12..2a140945f43 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -213,23 +213,23 @@ class NodeAllocation { * @return the final list of nodes */ List<Node> finalNodes(List<Node> surplusNodes) { - long currentRetired = nodes.stream().filter(node -> node.node.allocation().get().membership().retired()).count(); - long surplus = requestedNodes.surplusGiven(nodes.size()) - currentRetired; + int currentRetiredCount = (int) nodes.stream().filter(node -> node.node.allocation().get().membership().retired()).count(); + int deltaRetiredCount = requestedNodes.idealRetiredCount(nodes.size(), currentRetiredCount) - currentRetiredCount; - if (surplus > 0) { // retire until surplus is 0, prefer to retire higher indexes to minimize redistribution + if (deltaRetiredCount > 0) { // retire until deltaRetiredCount is 0, prefer to retire higher indexes to minimize redistribution for (PrioritizableNode node : byDecreasingIndex(nodes)) { if ( ! node.node.allocation().get().membership().retired() && node.node.state().equals(Node.State.active)) { node.node = node.node.retire(Agent.application, clock.instant()); surplusNodes.add(node.node); // offer this node to other groups - if (--surplus == 0) break; + if (--deltaRetiredCount == 0) break; } } } - else if (surplus < 0) { // unretire until surplus is 0 + else if (deltaRetiredCount < 0) { // unretire until deltaRetiredCount is 0 for (PrioritizableNode node : byIncreasingIndex(nodes)) { if ( node.node.allocation().get().membership().retired() && hasCompatibleFlavor(node.node)) { node.node = node.node.unretire(); - if (++surplus == 0) break; + if (++deltaRetiredCount == 0) break; } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index 23a6e3a8b9a..dc3f4a64421 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -34,8 +34,8 @@ public interface NodeSpec { /** Returns whether the given node count is sufficient to fulfill this spec */ boolean fulfilledBy(int count); - /** Returns the amount the given count is above the minimum amount needed to fulfill this request */ - int surplusGiven(int count); + /** Returns the ideal number of nodes that should be retired to fulfill this spec */ + int idealRetiredCount(int acceptedCount, int currentRetiredCount); /** Returns a specification of a fraction of all the nodes of this. It is assumed the argument is a valid divisor. */ NodeSpec fraction(int divisor); @@ -97,7 +97,7 @@ public interface NodeSpec { public boolean saturatedBy(int count) { return fulfilledBy(count); } // min=max for count specs @Override - public int surplusGiven(int count) { return count - this.count; } + public int idealRetiredCount(int acceptedCount, int currentRetiredCount) { return acceptedCount - this.count; } @Override public NodeSpec fraction(int divisor) { return new CountNodeSpec(count/divisor, requestedFlavor); } @@ -152,7 +152,14 @@ public interface NodeSpec { public boolean saturatedBy(int count) { return false; } @Override - public int surplusGiven(int count) { return 0; } + public int idealRetiredCount(int acceptedCount, int currentRetiredCount) { + /* + * All nodes marked with wantToRetire get marked as retired just before this function is called, + * the job of this function is to throttle the retired count. If no nodes are marked as retired + * then continue this way, otherwise allow only 1 node to be retired + */ + return Math.min(1, currentRetiredCount); + } @Override public NodeSpec fraction(int divisor) { return this; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java index 873193ac3b8..97fde2274f2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java @@ -11,15 +11,22 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.maintenance.JobControl; +import com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer; import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; +import org.junit.Before; import org.junit.Test; +import java.time.Duration; +import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; /** * Tests provisioning by node type instead of by count and flavor @@ -28,28 +35,31 @@ import static org.junit.Assert.assertFalse; */ public class NodeTypeProvisioningTest { - @Test - public void proxy_deployment() { - ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + private final ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + private final ApplicationId application = tester.makeApplicationId(); // application using proxy nodes + private final Capacity capacity = Capacity.fromRequiredNodeType(NodeType.proxy); + private final ClusterSpec clusterSpec = ClusterSpec.request( + ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42")); + @Before + public void setup() { tester.makeReadyNodes( 1, "small", NodeType.proxy); tester.makeReadyNodes( 3, "small", NodeType.host); tester.makeReadyNodes( 5, "small", NodeType.tenant); tester.makeReadyNodes(10, "large", NodeType.proxy); tester.makeReadyNodes(20, "large", NodeType.host); tester.makeReadyNodes(40, "large", NodeType.tenant); + } - ApplicationId application = tester.makeApplicationId(); // application using proxy nodes - - + @Test + public void proxy_deployment() { { // Deploy List<HostSpec> hosts = deployProxies(application, tester); assertEquals("Reserved all proxies", 11, hosts.size()); tester.activate(application, new HashSet<>(hosts)); List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); assertEquals("Activated all proxies", 11, nodes.size()); - for (Node node : nodes) - assertEquals(NodeType.proxy, node.type()); } { // Redeploy with no changes @@ -83,14 +93,178 @@ public class NodeTypeProvisioningTest { } } + @Test + public void retire_proxy() { + MockDeployer deployer = new MockDeployer( + tester.provisioner(), + Collections.singletonMap( + application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1))); + RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, + tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database())); + + { // Deploy + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals("Reserved all proxies", 11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals("Activated all proxies", 11, nodes.size()); + } + + Node nodeToRetire = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).get(5); + { // Pick out a node and retire it + tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true))); + + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(11, nodes.size()); + + // Verify that wantToRetire has been propagated + assertTrue(tester.nodeRepository().getNode(nodeToRetire.hostname()) + .flatMap(Node::allocation) + .map(allocation -> allocation.membership().retired()) + .orElseThrow(RuntimeException::new)); + } + + { // Redeploying while the node is still retiring has no effect + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(11, nodes.size()); + + // Verify that the node is still marked as retired + assertTrue(tester.nodeRepository().getNode(nodeToRetire.hostname()) + .flatMap(Node::allocation) + .map(allocation -> allocation.membership().retired()) + .orElseThrow(RuntimeException::new)); + } + + { + tester.advanceTime(Duration.ofMinutes(11)); + retiredExpirer.run(); + + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(10, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(10, nodes.size()); + + // Verify that the node is now inactive + assertEquals(Node.State.inactive, tester.nodeRepository().getNode(nodeToRetire.hostname()) + .orElseThrow(RuntimeException::new).state()); + } + } + + @Test + public void retire_multiple_proxy_simultaneously() { + MockDeployer deployer = new MockDeployer( + tester.provisioner(), + Collections.singletonMap( + application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1))); + RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, + tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database())); + final int numNodesToRetire = 5; + + { // Deploy + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals("Reserved all proxies", 11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals("Activated all proxies", 11, nodes.size()); + } + + List<Node> nodesToRetire = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active) + .subList(3, 3 + numNodesToRetire); + String currentyRetiringHostname; + { + nodesToRetire.forEach(nodeToRetire -> + tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true)))); + + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(11, nodes.size()); + + // Verify that wantToRetire has been propagated + List<Node> nodesCurrentlyRetiring = nodes.stream() + .filter(node -> node.allocation().get().membership().retired()) + .collect(Collectors.toList()); + assertEquals(1, nodesCurrentlyRetiring.size()); + + // The retiring node should be one of the nodes we marked for retirement + currentyRetiringHostname = nodesCurrentlyRetiring.get(0).hostname(); + assertTrue(nodesToRetire.stream().map(Node::hostname).filter(hostname -> hostname.equals(currentyRetiringHostname)).count() == 1); + } + + { // Redeploying while the node is still retiring has no effect + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(11, nodes.size()); + + // Verify that wantToRetire has been propagated + List<Node> nodesCurrentlyRetiring = nodes.stream() + .filter(node -> node.allocation().get().membership().retired()) + .collect(Collectors.toList()); + assertEquals(1, nodesCurrentlyRetiring.size()); + + // The node that started retiring is still the only one retiring + assertEquals(currentyRetiringHostname, nodesCurrentlyRetiring.get(0).hostname()); + } + + { + tester.advanceTime(Duration.ofMinutes(11)); + retiredExpirer.run(); + + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(10, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(10, nodes.size()); + + // Verify the node we previously set to retire has finished retiring + assertEquals(Node.State.inactive, tester.nodeRepository().getNode(currentyRetiringHostname) + .orElseThrow(RuntimeException::new).state()); + + // Verify that a node is currently retiring + List<Node> nodesCurrentlyRetiring = nodes.stream() + .filter(node -> node.allocation().get().membership().retired()) + .collect(Collectors.toList()); + assertEquals(1, nodesCurrentlyRetiring.size()); + + // This node is different from the one that was retiring previously + String newRetiringHostname = nodesCurrentlyRetiring.get(0).hostname(); + assertNotEquals(currentyRetiringHostname, newRetiringHostname); + // ... but is one of the nodes that were put to wantToRetire earlier + assertTrue(nodesToRetire.stream().map(Node::hostname).filter(hostname -> hostname.equals(newRetiringHostname)).count() == 1); + } + + + for (int i = 0; i < 10; i++){ + tester.advanceTime(Duration.ofMinutes(11)); + retiredExpirer.run(); + List<HostSpec> hosts = deployProxies(application, tester); + tester.activate(application, new HashSet<>(hosts)); + } + + // After a long time, all currently active proxy nodes are not marked with wantToRetire or as retired + long numRetiredActiveProxyNodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).stream() + .filter(node -> !node.status().wantToRetire()) + .filter(node -> !node.allocation().get().membership().retired()) + .count(); + assertEquals(11 - numNodesToRetire, numRetiredActiveProxyNodes); + + // All the nodes that were marked with wantToRetire earlier are now inactive + assertEquals(nodesToRetire.stream().map(Node::hostname).collect(Collectors.toSet()), + tester.nodeRepository().getNodes(Node.State.inactive).stream().map(Node::hostname).collect(Collectors.toSet())); + } + private List<HostSpec> deployProxies(ApplicationId application, ProvisioningTester tester) { - return tester.prepare(application, - ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from("test"), - Version.fromString("6.42")), - Capacity.fromRequiredNodeType(NodeType.proxy), - 1); - + return tester.prepare(application, clusterSpec, capacity, 1); } } diff --git a/parent/pom.xml b/parent/pom.xml index ba03ae0d924..ccbe34841fc 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -692,6 +692,11 @@ <artifactId>athenz-zts-java-client</artifactId> <version>${athenz.version}</version> </dependency> + <dependency> + <groupId>com.github.tomakehurst</groupId> + <artifactId>wiremock-standalone</artifactId> + <version>2.6.0</version> + </dependency> </dependencies> </dependencyManagement> @@ -702,7 +707,7 @@ <aries.util.version>1.0.0</aries.util.version> <asm-debug-all.version>5.0.3</asm-debug-all.version> <!-- Athenz dependencies. Make sure these dependencies matches those in Vespa's internal repositories --> - <athenz.version>1.7.28</athenz.version> + <athenz.version>1.7.41</athenz.version> <bouncycastle.version>1.58</bouncycastle.version> <commons-lang.version>2.6</commons-lang.version> <!-- WARNING: If you change curator version, you also need to update diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 9a4f094d32c..84adaa574ec 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -4,6 +4,7 @@ #include <vespa/document/select/parser.h> #include <vespa/document/base/documentid.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/vespalib/util/crc.h> #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -17,6 +18,7 @@ LOG_SETUP(".dummypersistence"); using vespalib::make_string; using std::binary_search; using std::lower_bound; +using document::FixedBucketSpaces; namespace storage::spi::dummy { @@ -341,16 +343,18 @@ DummyPersistence::getPartitionStates() const BucketIdListResult -DummyPersistence::listBuckets(BucketSpace, PartitionId id) const +DummyPersistence::listBuckets(BucketSpace bucketSpace, PartitionId id) const { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "listBuckets(%u)", uint16_t(id)); vespalib::MonitorGuard lock(_monitor); BucketIdListResult::List list; - for (PartitionContent::const_iterator it = _content[id].begin(); - it != _content[id].end(); ++it) - { - list.push_back(it->first); + if (bucketSpace == FixedBucketSpaces::default_space()) { + for (PartitionContent::const_iterator it = _content[id].begin(); + it != _content[id].end(); ++it) + { + list.push_back(it->first); + } } return BucketIdListResult(list); } @@ -363,23 +367,30 @@ DummyPersistence::setModifiedBuckets(const BucketIdListResult::List& buckets) } BucketIdListResult -DummyPersistence::getModifiedBuckets(BucketSpace) const +DummyPersistence::getModifiedBuckets(BucketSpace bucketSpace) const { vespalib::MonitorGuard lock(_monitor); - return BucketIdListResult(_modifiedBuckets); + if (bucketSpace == FixedBucketSpaces::default_space()) { + return BucketIdListResult(_modifiedBuckets); + } else { + BucketIdListResult::List emptyList; + return BucketIdListResult(emptyList); + } } Result -DummyPersistence::setClusterState(BucketSpace, const ClusterState& c) +DummyPersistence::setClusterState(BucketSpace bucketSpace, const ClusterState& c) { vespalib::MonitorGuard lock(_monitor); - _clusterState.reset(new ClusterState(c)); - if (!_clusterState->nodeUp()) { - for (uint32_t i=0, n=_content.size(); i<n; ++i) { - for (PartitionContent::iterator it = _content[i].begin(); - it != _content[i].end(); ++it) - { - it->second->setActive(false); + if (bucketSpace == FixedBucketSpaces::default_space()) { + _clusterState.reset(new ClusterState(c)); + if (!_clusterState->nodeUp()) { + for (uint32_t i=0, n=_content.size(); i<n; ++i) { + for (PartitionContent::iterator it = _content[i].begin(); + it != _content[i].end(); ++it) + { + it->second->setActive(false); + } } } } @@ -394,6 +405,7 @@ DummyPersistence::setActiveState(const Bucket& b, LOG(debug, "setCurrentState(%s, %s)", b.toString().c_str(), newState == BucketInfo::ACTIVE ? "ACTIVE" : "INACTIVE"); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { @@ -407,6 +419,7 @@ BucketInfoResult DummyPersistence::getBucketInfo(const Bucket& b) const { DUMMYPERSISTENCE_VERIFY_INITIALIZED; + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { LOG(debug, "getBucketInfo(%s) : (bucket not found)", @@ -430,6 +443,7 @@ DummyPersistence::put(const Bucket& b, Timestamp t, const Document::SP& doc, b.toString().c_str(), uint64_t(t), doc->getId().toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { return BucketInfoResult(Result::TRANSIENT_ERROR, "Bucket not found"); @@ -456,6 +470,7 @@ Result DummyPersistence::maintain(const Bucket& b, MaintenanceLevel) { + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); if (_simulateMaintainFailure) { BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { @@ -484,6 +499,7 @@ DummyPersistence::remove(const Bucket& b, b.toString().c_str(), uint64_t(t), did.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { @@ -511,6 +527,7 @@ DummyPersistence::get(const Bucket& b, LOG(debug, "get(%s, %s)", b.toString().c_str(), did.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { } else { @@ -538,6 +555,7 @@ DummyPersistence::createIterator( { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "createIterator(%s)", b.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); std::unique_ptr<document::select::Node> docSelection; if (!s.getDocumentSelection().getDocumentSelection().empty()) { docSelection.reset( @@ -712,6 +730,7 @@ DummyPersistence::createBucket(const Bucket& b, Context&) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "createBucket(%s)", b.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); vespalib::MonitorGuard lock(_monitor); if (_content[b.getPartition()].find(b) == _content[b.getPartition()].end()) { _content[b.getPartition()][b] = std::make_shared<BucketContent>(); @@ -727,6 +746,7 @@ DummyPersistence::deleteBucket(const Bucket& b, Context&) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "deleteBucket(%s)", b.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); vespalib::MonitorGuard lock(_monitor); if (_content[b.getPartition()][b].get()) { assert(!_content[b.getPartition()][b]->_inUse); @@ -746,6 +766,9 @@ DummyPersistence::split(const Bucket& source, source.toString().c_str(), target1.toString().c_str(), target2.toString().c_str()); + assert(source.getBucketSpace() == FixedBucketSpaces::default_space()); + assert(target1.getBucketSpace() == FixedBucketSpaces::default_space()); + assert(target2.getBucketSpace() == FixedBucketSpaces::default_space()); createBucket(source, context); createBucket(target1, context); createBucket(target2, context); @@ -799,6 +822,9 @@ DummyPersistence::join(const Bucket& source1, const Bucket& source2, source1.toString().c_str(), source2.toString().c_str(), target.toString().c_str()); + assert(source1.getBucketSpace() == FixedBucketSpaces::default_space()); + assert(source2.getBucketSpace() == FixedBucketSpaces::default_space()); + assert(target.getBucketSpace() == FixedBucketSpaces::default_space()); createBucket(target, context); BucketContentGuard::UP targetGuard(acquireBucketWithLock(target)); assert(targetGuard.get()); @@ -833,6 +859,7 @@ DummyPersistence::revert(const Bucket& b, Timestamp t, Context&) LOG(debug, "revert(%s, %zu)", b.toString().c_str(), uint64_t(t)); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { @@ -883,6 +910,7 @@ DummyPersistence::dumpBucket(const Bucket& b) const { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(spam, "dumpBucket(%s)", b.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); vespalib::MonitorGuard lock(_monitor); PartitionContent::const_iterator it(_content[b.getPartition()].find(b)); if (it == _content[b.getPartition()].end()) { @@ -902,6 +930,7 @@ bool DummyPersistence::isActive(const Bucket& b) const { DUMMYPERSISTENCE_VERIFY_INITIALIZED; + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); vespalib::MonitorGuard lock(_monitor); LOG(spam, "isActive(%s)", b.toString().c_str()); PartitionContent::const_iterator it(_content[b.getPartition()].find(b)); @@ -919,6 +948,7 @@ BucketContentGuard::~BucketContentGuard() BucketContentGuard::UP DummyPersistence::acquireBucketWithLock(const Bucket& b) const { + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); vespalib::MonitorGuard lock(_monitor); DummyPersistence& ncp(const_cast<DummyPersistence&>(*this)); PartitionContent::iterator it(ncp._content[b.getPartition()].find(b)); diff --git a/protocols/getnodestate/distributor.json b/protocols/getnodestate/distributor.json index f4a0ab42243..ceed1a1b5c3 100644 --- a/protocols/getnodestate/distributor.json +++ b/protocols/getnodestate/distributor.json @@ -9,7 +9,19 @@ "latency-ms-sum": 10000, "count": 3 } - } + }, + "bucket-spaces" : [ + { + "name": "default", + "total": 11, + "pending": 3 + }, + { + "name": "global", + "total": 13, + "pending": 5 + } + ] }, { "node-index": 5, @@ -19,7 +31,14 @@ "latency-ms-sum": 25000, "count": 7 } - } + }, + "bucket-spaces" : [ + { + "name": "default", + "total": 17, + "pending": 7 + } + ] } ] } diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index a0f34072f06..0e1bad796f5 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -16,7 +16,6 @@ LOG_SETUP("vespa-gen-testdocs"); typedef vespalib::hash_set<vespalib::string> StringSet; -typedef vespalib::hash_set<uint32_t> UIntSet; typedef std::vector<vespalib::string> StringArray; typedef std::shared_ptr<StringArray> StringArraySP; using namespace vespalib::alloc; @@ -157,17 +156,6 @@ public: setup(); virtual void - clear(); - - virtual void - deleteHistogram(const string &baseDir, - const string &name); - - virtual void - writeHistogram(const string &baseDir, - const string &name); - - virtual void generate(vespalib::asciistream &doc, uint32_t id) = 0; }; @@ -189,38 +177,11 @@ FieldGenerator::setup() } -void -FieldGenerator::clear() -{ -} - - -void -FieldGenerator::deleteHistogram(const string &baseDir, - const string &name) -{ - (void) baseDir; - (void) name; -} - - -void -FieldGenerator::writeHistogram(const string &baseDir, - const string &name) -{ - (void) baseDir; - (void) name; -} - - class RandTextFieldGenerator : public FieldGenerator { search::Rand48 &_rnd; uint32_t _numWords; StringArray _strings; - std::vector<uint32_t> _histogram; - UIntSet _wnums; - uint32_t _colls; uint32_t _minFill; uint32_t _randFill; @@ -238,15 +199,6 @@ public: setup() override; virtual void - clear() override; - - virtual void - deleteHistogram(const string &baseDir, const string &name) override; - - virtual void - writeHistogram(const string &baseDir, const string &name) override; - - virtual void generate(vespalib::asciistream &doc, uint32_t id) override; }; @@ -260,9 +212,6 @@ RandTextFieldGenerator::RandTextFieldGenerator(const string &name, _rnd(rnd), _numWords(numWords), _strings(), - _histogram(), - _wnums(), - _colls(0u), _minFill(minFill), _randFill(randFill) { @@ -282,51 +231,6 @@ RandTextFieldGenerator::setup() "generating dictionary for field %s (%u words)", _name.c_str(), _numWords); StringGenerator(_rnd).rand_unique_array(_strings, 5, 10, _numWords); - _histogram.resize(_numWords); -} - - -void -RandTextFieldGenerator::clear() -{ - typedef std::vector<uint32_t>::iterator HI; - for (HI i(_histogram.begin()), ie(_histogram.end()); i != ie; ++i) { - *i = 0; - } - _colls = 0; -} - - -void -RandTextFieldGenerator::deleteHistogram(const string &baseDir, - const string &name) -{ - string fname(prependBaseDir(baseDir, name) + "-" + _name); - FastOS_File::Delete(fname.c_str()); -} - - -void -RandTextFieldGenerator::writeHistogram(const string &baseDir, - const string &name) -{ - LOG(info, "%u word collisions for field %s", _colls, _name.c_str()); - string fname(name + "-" + _name); - string fullName(prependBaseDir(baseDir, fname)); - LOG(info, "Writing histogram %s", fname.c_str()); - Fast_BufferedFile f(new FastOS_File); - f.WriteOpen(fullName.c_str()); - uint32_t numWords = _strings.size(); - assert(numWords == _histogram.size()); - for (uint32_t wNum = 0; wNum < numWords; ++wNum) { - f.WriteString(_strings[wNum].c_str()); - f.WriteString(" "); - f.addNum(_histogram[wNum], 0, ' '); - f.WriteString("\n"); - } - f.Flush(); - f.Close(); - shafile(baseDir, fname); } @@ -335,7 +239,6 @@ RandTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id) { (void) id; doc << " <" << _name << ">"; - _wnums.clear(); uint32_t gLen = _minFill + _rnd.lrand48() % (_randFill + 1); bool first = true; for (uint32_t i = 0; i < gLen; ++i) { @@ -343,10 +246,6 @@ RandTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id) doc << " "; first = false; uint32_t wNum = _rnd.lrand48() % _strings.size(); - if (_wnums.insert(wNum).second) - _histogram[wNum]++; - else - ++_colls; const string &s(_strings[wNum]); assert(s.size() > 0); doc << s; @@ -369,12 +268,6 @@ public: ~ModTextFieldGenerator(); virtual void - clear() override; - - virtual void - writeHistogram(const string &name); - - virtual void generate(vespalib::asciistream &doc, uint32_t id) override; }; @@ -395,19 +288,6 @@ ModTextFieldGenerator::~ModTextFieldGenerator() void -ModTextFieldGenerator::clear() -{ -} - - -void -ModTextFieldGenerator::writeHistogram(const string &name) -{ - (void) name; -} - - -void ModTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id) { typedef std::vector<uint32_t>::const_iterator MI; @@ -433,12 +313,6 @@ public: ~IdTextFieldGenerator(); virtual void - clear() override; - - virtual void - writeHistogram(const string &name); - - virtual void generate(vespalib::asciistream &doc, uint32_t id) override; }; @@ -455,19 +329,6 @@ IdTextFieldGenerator::~IdTextFieldGenerator() void -IdTextFieldGenerator::clear() -{ -} - - -void -IdTextFieldGenerator::writeHistogram(const string &name) -{ - (void) name; -} - - -void IdTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id) { doc << " <" << _name << ">"; @@ -492,12 +353,6 @@ public: ~RandIntFieldGenerator(); virtual void - clear() override; - - virtual void - writeHistogram(const string &name); - - virtual void generate(vespalib::asciistream &doc, uint32_t id) override; }; @@ -521,19 +376,6 @@ RandIntFieldGenerator::~RandIntFieldGenerator() void -RandIntFieldGenerator::clear() -{ -} - - -void -RandIntFieldGenerator::writeHistogram(const string &name) -{ - (void) name; -} - - -void RandIntFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id) { (void) id; @@ -561,17 +403,6 @@ public: ~DocumentGenerator(); void - clear(); - - void - deleteHistogram(const string &baseDir, - const string &name); - - void - writeHistogram(const string &baseDir, - const string &name); - - void generate(uint32_t id); void @@ -609,16 +440,6 @@ DocumentGenerator::setup() void -DocumentGenerator::clear() -{ - typedef FieldVec::const_iterator FI; - for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) { - (*i)->clear(); - } -} - - -void DocumentGenerator::generate(uint32_t id) { _doc.clear(); @@ -633,26 +454,6 @@ DocumentGenerator::generate(uint32_t id) void -DocumentGenerator::deleteHistogram(const string &baseDir, - const string &name) -{ - typedef FieldVec::const_iterator FI; - for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) { - (*i)->deleteHistogram(baseDir, name); - } -} - -void -DocumentGenerator::writeHistogram(const string &baseDir, - const string &name) -{ - typedef FieldVec::const_iterator FI; - for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) { - (*i)->writeHistogram(baseDir, name); - } -} - -void DocumentGenerator::generate(uint32_t docMin, uint32_t docCount, const string &baseDir, const string &feedFileName, @@ -660,11 +461,8 @@ DocumentGenerator::generate(uint32_t docMin, uint32_t docCount, { string fullName(prependBaseDir(baseDir, feedFileName)); FastOS_File::Delete(fullName.c_str()); - string histname(feedFileName + ".histogram"); - deleteHistogram(baseDir, histname); Fast_BufferedFile f(new FastOS_File); f.WriteOpen(fullName.c_str()); - clear(); if (headers) { f.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n"); f.WriteString("<vespafeed>\n"); @@ -681,7 +479,6 @@ DocumentGenerator::generate(uint32_t docMin, uint32_t docCount, f.Close(); LOG(info, "Calculating sha256 for %s", feedFileName.c_str()); shafile(baseDir, feedFileName); - writeHistogram(baseDir, histname); } diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp index 98642c6edf6..151b1e1596b 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp @@ -869,6 +869,8 @@ FastS_FNET_Search::CheckCoverage() uint64_t activeDocs = 0; uint64_t soonActiveDocs = 0; uint32_t degradedReason = 0; + uint16_t nodesQueried = 0; + uint16_t nodesReplied = 0; size_t cntNone(0); for (const FastS_FNET_SearchNode & node : _nodes) { @@ -877,14 +879,17 @@ FastS_FNET_Search::CheckCoverage() activeDocs += node._qresult->_activeDocs; soonActiveDocs += node._qresult->_soonActiveDocs; degradedReason |= node._qresult->_coverageDegradeReason; + nodesQueried += node._qresult->getNodesQueried(); + nodesReplied += node._qresult->getNodesReplied(); } else { + nodesQueried++; cntNone++; } } if ((cntNone > 0) && (cntNone != _nodes.size())) { activeDocs += cntNone * activeDocs/(_nodes.size() - cntNone); } - _util.SetCoverage(covDocs, activeDocs, soonActiveDocs, degradedReason, _nodes.size(), _nodes.size() - cntNone); + _util.SetCoverage(covDocs, activeDocs, soonActiveDocs, degradedReason, nodesQueried, nodesReplied); } @@ -1151,7 +1156,7 @@ FastS_FNET_Search::setupQueryPacket(uint32_t hitsPerNode, uint32_t qflags, qx._features = search::fs4transport::QF_PARSEDQUERY | search::fs4transport::QF_RANKP; qx._offset = _util.GetAlignedSearchOffset(); qx._maxhits = hitsPerNode; // capped maxhits - qx._qflags = qflags; // filtered query flags + qx.setQueryFlags(qflags); qx.setTimeout(_queryArgs->getTimeLeft()); qx.setRanking(_queryArgs->ranking); diff --git a/searchlib/src/tests/common/packets/packets_test.cpp b/searchlib/src/tests/common/packets/packets_test.cpp index ad4cf02f4e8..35ca7bc1dd9 100644 --- a/searchlib/src/tests/common/packets/packets_test.cpp +++ b/searchlib/src/tests/common/packets/packets_test.cpp @@ -352,8 +352,8 @@ TEST("testQueryResultX") { EXPECT_EQUAL(2u, ptr->_totNumDocs); EXPECT_EQUAL((search::HitRank)3, ptr->_maxRank); EXPECT_EQUAL(4u, ptr->getDistributionKey()); - EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 12 : 0u, ptr->getNodesQueried()); - EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 11 : 0u, ptr->getNodesReplied()); + EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 12 : 1u, ptr->getNodesQueried()); + EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 11 : 1u, ptr->getNodesReplied()); EXPECT_EQUAL(6u, ptr->_coverageDocs); EXPECT_EQUAL(7u, ptr->_activeDocs); EXPECT_EQUAL(8u, ptr->_soonActiveDocs); @@ -398,7 +398,7 @@ createAndFill_QUERYX() EXPECT_EQUAL(0l, src->getTimeout()); src->setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); EXPECT_EQUAL(fastos::TimeStamp(4*fastos::TimeStamp::MS), src->getTimeout()); - src->_qflags = 5u; + src->setQueryFlags(5u); src->setRanking("seven"); src->_numStackItems = 14u; src->_propsVector.resize(2); @@ -419,7 +419,7 @@ verifyQueryX(FS4Packet_QUERYX & queryX, uint32_t features) EXPECT_EQUAL(2u, queryX._offset); EXPECT_EQUAL(3u, queryX._maxhits); EXPECT_EQUAL(fastos::TimeStamp(4*fastos::TimeStamp::MS), queryX.getTimeout()); - EXPECT_EQUAL(0x5u, queryX._qflags); + EXPECT_EQUAL(0x1u, queryX.getQueryFlags()); //Filtered if (queryX._features & QF_RANKP) { EXPECT_EQUAL("seven", queryX._ranking); } else { diff --git a/searchlib/src/tests/engine/searchapi/searchapi_test.cpp b/searchlib/src/tests/engine/searchapi/searchapi_test.cpp index 3e8be3a99fc..a517890620c 100644 --- a/searchlib/src/tests/engine/searchapi/searchapi_test.cpp +++ b/searchlib/src/tests/engine/searchapi/searchapi_test.cpp @@ -54,7 +54,7 @@ Test::convertToRequest() src._offset = 2u; src._maxhits = 3u; src.setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); - src._qflags = 5u; + src.setQueryFlags(5u); src._features |= QF_RANKP; src.setRanking("seven"); src._features |= QF_PROPERTIES; @@ -92,7 +92,7 @@ Test::convertToRequest() EXPECT_EQUAL(dst.offset, 2u); EXPECT_EQUAL(dst.maxhits, 3u); EXPECT_EQUAL((dst.getTimeOfDoom() - dst.getStartTime()).ms(), 4u); - EXPECT_EQUAL(dst.queryFlags, 5u); + EXPECT_EQUAL(dst.queryFlags, 1u); //Filtered EXPECT_EQUAL(vespalib::string("seven"), dst.ranking); EXPECT_EQUAL(dst.propertiesMap.size(), 2u); EXPECT_EQUAL(dst.propertiesMap.featureOverrides().lookup("p1k1").get(), std::string("p1v1")); diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 4bb11acaabb..5004ba80913 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -906,8 +906,8 @@ FS4Packet_QUERYRESULTX::AllocateHits(uint32_t cnt) FS4Packet_QUERYRESULTX::FS4Packet_QUERYRESULTX() : FS4Packet(), _distributionKey(0), - _nodesQueried(0), - _nodesReplied(0), + _nodesQueried(1), + _nodesReplied(1), _features(QRF_COVERAGE | QRF_EXTENDED_COVERAGE), _offset(0), _numDocs(0), @@ -1172,10 +1172,10 @@ FS4Packet_QUERYRESULTX::toString(uint32_t indent) const FS4Packet_QUERYX::FS4Packet_QUERYX() : FS4Packet(), _timeout(0), + _qflags(0), _features(0), _offset(0), _maxhits(0), - _qflags(0), _ranking(), _propsVector(), _sortSpec(), diff --git a/searchlib/src/vespa/searchlib/common/packets.h b/searchlib/src/vespa/searchlib/common/packets.h index a876400e503..87e2bd998b1 100644 --- a/searchlib/src/vespa/searchlib/common/packets.h +++ b/searchlib/src/vespa/searchlib/common/packets.h @@ -42,6 +42,12 @@ enum fnet_feature_masks { GDF_RESCLASSNAME | GDF_PROPERTIES | GDF_FLAGS), + ACTIVE_QUERY_FLAGS = (QFLAG_EXTENDED_COVERAGE | + QFLAG_COVERAGE_NODES | + QFLAG_ESTIMATE | + QFLAG_DROP_SORTDATA | + QFLAG_NO_RESULTCACHE | + QFLAG_DUMP_FEATURES), FNET_MQF_SUPPORTED_MASK = (MQF_QFLAGS), @@ -441,12 +447,12 @@ private: FS4Packet_QUERYX& operator=(const FS4Packet_QUERYX &); uint32_t _timeout; + uint32_t _qflags; public: uint32_t _features; // see query_features uint32_t _offset; uint32_t _maxhits; - uint32_t _qflags; string _ranking; // if QF_RANKP PropsVector _propsVector; // if QF_PROPERTIES string _sortSpec; // if QF_SORTSPEC @@ -457,6 +463,7 @@ public: uint32_t _numStackItems; // if QF_PARSEDQUERY string _stackDump; // if QF_PARSEDQUERY + void setQueryFlags(uint32_t qflags) { _qflags = ACTIVE_QUERY_FLAGS & qflags; } void setRanking(const vespalib::stringref &ranking) { _ranking = ranking; } void setSortSpec(const vespalib::stringref &spec) { _sortSpec = spec; } void setGroupSpec(const vespalib::stringref &spec) { _groupSpec = spec; } @@ -465,6 +472,7 @@ public: void setStackDump(const vespalib::stringref &buf) { _stackDump = buf; } void setTimeout(const fastos::TimeStamp & timeout); fastos::TimeStamp getTimeout() const; + uint32_t getQueryFlags() const { return _qflags; } explicit FS4Packet_QUERYX(); ~FS4Packet_QUERYX(); diff --git a/searchlib/src/vespa/searchlib/common/transport.h b/searchlib/src/vespa/searchlib/common/transport.h index b6a925240c9..55cb44a34d0 100644 --- a/searchlib/src/vespa/searchlib/common/transport.h +++ b/searchlib/src/vespa/searchlib/common/transport.h @@ -12,6 +12,7 @@ namespace search::fs4transport { * are as follows: * <ul> * <li><b>QFLAG_EXTENDED_COVERAGE</b>: Indicates that the it is able to receive extended coverage information.</li> + * <li><b>QFLAG_COVERAGE_NODES</b>: Indicate that it is able to handle nodes information.</li> * <li><b>QFLAG_ESTIMATE</b>: Indicates that the query is performed to get * an estimate of the total number of hits</li> * <li><b>QFLAG_DUMP_FEATURES</b>: Dump detailed ranking information. Note that diff --git a/searchlib/src/vespa/searchlib/engine/packetconverter.cpp b/searchlib/src/vespa/searchlib/engine/packetconverter.cpp index 2ebbb9bf89c..e6fb37223d6 100644 --- a/searchlib/src/vespa/searchlib/engine/packetconverter.cpp +++ b/searchlib/src/vespa/searchlib/engine/packetconverter.cpp @@ -55,7 +55,7 @@ PacketConverter::toSearchRequest(const QUERYX &packet, SearchRequest &request) request.offset = packet._offset; request.maxhits = packet._maxhits; request.setTimeout(packet.getTimeout()); - request.queryFlags = packet._qflags; + request.queryFlags = packet.getQueryFlags(); request.ranking = packet._ranking; for (uint32_t i = 0; i < packet._propsVector.size(); ++i) { diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigInstallVariables.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigInstallVariables.scala new file mode 100644 index 00000000000..8cfa01937c9 --- /dev/null +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigInstallVariables.scala @@ -0,0 +1,91 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + +import java.util.Optional + +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions.ConfigServer + +import scala.language.implicitConversions +import scala.util.Try + +/** + * @author Tony Vaagenes + */ +class CloudConfigInstallVariables extends CloudConfigOptions { + import CloudConfigInstallVariables._ + + override val rpcPort = optionalInstallVar[Integer]("port_configserver_rpc", "services") + override val allConfigServers = installVar("addr_configserver", "services") withDefault Array[ConfigServer]() + override val multiTenant = optionalInstallVar[java.lang.Boolean]("multitenant") + + override val zookeeperBarrierTimeout = optionalInstallVar[java.lang.Long]("zookeeper_barrier_timeout") + override val sessionLifeTimeSecs = optionalInstallVar[java.lang.Long]("session_lifetime") + override val configModelPluginDirs = installVar("config_model_plugin_dirs") withDefault Array[String]() + override val zookeeperClientPort = optionalInstallVar[Integer]("zookeeper_clientPort") + override val zookeeperQuorumPort = optionalInstallVar[Integer]("zookeeper_quoromPort") + override val zookeeperElectionPort = optionalInstallVar[Integer]("zookeeper_electionPort") + override val payloadCompressionType = optionalInstallVar[java.lang.String]("payload_compression_type") + override val environment = optionalInstallVar[java.lang.String]("environment") + override val region = optionalInstallVar[java.lang.String]("region") + override val system = optionalInstallVar[java.lang.String]("system") + override val defaultFlavor = optionalInstallVar[java.lang.String]("default_flavor") + override val defaultAdminFlavor = optionalInstallVar[java.lang.String]("default_admin_flavor") + override val defaultContainerFlavor = optionalInstallVar[java.lang.String]("default_container_flavor") + override val defaultContentFlavor = optionalInstallVar[java.lang.String]("default_content_flavor") + override val useVespaVersionInRequest = optionalInstallVar[java.lang.Boolean]("use_vespa_version_in_request") + override val hostedVespa = optionalInstallVar[java.lang.Boolean]("hosted_vespa") + override val numParallelTenantLoaders = optionalInstallVar[java.lang.Integer]("num_parallel_tenant_loaders") + override val dockerRegistry = optionalInstallVar[java.lang.String]("docker_registry") + override val dockerVespaBaseImage = optionalInstallVar[java.lang.String]("docker_vespa_base_image") + override val loadBalancerAddress = optionalInstallVar[java.lang.String]("load_balancer_address") +} + +object CloudConfigInstallVariables { + private class InstallVariable(installPkg:String, name: String) { + val value = Environment.optionalInstallVariable(installPkg + "." + name) + + def withDefault[T](defaultValue: T)(implicit c: Converter[T]) : T = { + value map { implicitly[Converter[T]].convert } getOrElse defaultValue + } + } + + private def installVar(setting:String, installPkg: String = "cloudconfig_server") = new InstallVariable(installPkg, setting) + + private def optionalInstallVar[T](setting:String, installPkg: String = "cloudconfig_server")(implicit c: Converter[T]): Optional[T] = { + Environment.optionalInstallVariable(installPkg + "." + setting) map ( c.convert ) + } + + implicit val configServerConverter: Converter[Array[ConfigServer]] = new Converter[Array[ConfigServer]] { + override def convert(s: String) = { + s split "[, ]" filter { !_.isEmpty } map { toConfigServer } + } + } + + implicit val stringArrayConverter: Converter[Array[String]] = new Converter[Array[String]] { + override def convert(s: String) = { + s split "[, ]" filter { !_.isEmpty } + } + } + + private def toConfigServer(hostPort: String): ConfigServer = Try { + val (host, portStr) = splitFirst(hostPort, ':') + val port = portStr map { _.toInt } + new ConfigServer(host, port) + }.getOrElse(throw new IllegalArgumentException(s"Invalid config server '$hostPort'")) + + private def splitFirst(string: String, separator: Character): (String, Option[String]) = { + val (beginning, endWithSeparator) = string span { _ != separator } + (beginning, tailOption(endWithSeparator)) + } + + def tailOption(s: String) = { + if (s.isEmpty) None + else Some(s.tail) + } + + implicit def toJavaOptional[U <% V, V](option: Option[U]): Optional[V] = option match { + case Some(u) => Optional.of(u: V) + case None => Optional.empty() + } +} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala deleted file mode 100644 index 41e6b66b986..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import java.util.Optional - -import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions -import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions.ConfigServer - -import scala.language.implicitConversions -import scala.util.Try - -/** - * @author Tony Vaagenes - */ -class CloudConfigYinstVariables extends CloudConfigOptions { - import CloudConfigYinstVariables._ - - override val rpcPort = optionalYinstVar[Integer]("port_configserver_rpc", "services") - override val allConfigServers = yinstVar("addr_configserver", "services") withDefault Array[ConfigServer]() - override val multiTenant = optionalYinstVar[java.lang.Boolean]("multitenant") - - override val zookeeperBarrierTimeout = optionalYinstVar[java.lang.Long]("zookeeper_barrier_timeout") - override val sessionLifeTimeSecs = optionalYinstVar[java.lang.Long]("session_lifetime") - override val configModelPluginDirs = yinstVar("config_model_plugin_dirs") withDefault Array[String]() - override val zookeeperClientPort = optionalYinstVar[Integer]("zookeeper_clientPort") - override val zookeeperQuorumPort = optionalYinstVar[Integer]("zookeeper_quoromPort") - override val zookeeperElectionPort = optionalYinstVar[Integer]("zookeeper_electionPort") - override val payloadCompressionType = optionalYinstVar[java.lang.String]("payload_compression_type") - override val environment = optionalYinstVar[java.lang.String]("environment") - override val region = optionalYinstVar[java.lang.String]("region") - override val system = optionalYinstVar[java.lang.String]("system") - override val defaultFlavor = optionalYinstVar[java.lang.String]("default_flavor") - override val defaultAdminFlavor = optionalYinstVar[java.lang.String]("default_admin_flavor") - override val defaultContainerFlavor = optionalYinstVar[java.lang.String]("default_container_flavor") - override val defaultContentFlavor = optionalYinstVar[java.lang.String]("default_content_flavor") - override val useVespaVersionInRequest = optionalYinstVar[java.lang.Boolean]("use_vespa_version_in_request") - override val hostedVespa = optionalYinstVar[java.lang.Boolean]("hosted_vespa") - override val numParallelTenantLoaders = optionalYinstVar[java.lang.Integer]("num_parallel_tenant_loaders") - override val dockerRegistry = optionalYinstVar[java.lang.String]("docker_registry") - override val dockerVespaBaseImage = optionalYinstVar[java.lang.String]("docker_vespa_base_image") - override val loadBalancerAddress = optionalYinstVar[java.lang.String]("load_balancer_address") - override val disableFiledistributor = optionalYinstVar[java.lang.Boolean]("disable_filedistributor") -} - -object CloudConfigYinstVariables { - private class YinstVariable(yinstPkg:String, name: String) { - val value = Environment.optionalYinstVariable(yinstPkg + "." + name) - - def withDefault[T](defaultValue: T)(implicit c: Converter[T]) : T = { - value map { implicitly[Converter[T]].convert } getOrElse defaultValue - } - } - - private def yinstVar(setting:String, yinstPkg: String = "cloudconfig_server") = new YinstVariable(yinstPkg, setting) - - private def optionalYinstVar[T](setting:String, yinstPkg: String = "cloudconfig_server")(implicit c: Converter[T]): Optional[T] = { - Environment.optionalYinstVariable(yinstPkg + "." + setting) map ( c.convert ) - } - - implicit val configServerConverter: Converter[Array[ConfigServer]] = new Converter[Array[ConfigServer]] { - override def convert(s: String) = { - s split "[, ]" filter { !_.isEmpty } map { toConfigServer } - } - } - - implicit val stringArrayConverter: Converter[Array[String]] = new Converter[Array[String]] { - override def convert(s: String) = { - s split "[, ]" filter { !_.isEmpty } - } - } - - private def toConfigServer(hostPort: String): ConfigServer = Try { - val (host, portStr) = splitFirst(hostPort, ':') - val port = portStr map { _.toInt } - new ConfigServer(host, port) - }.getOrElse(throw new IllegalArgumentException(s"Invalid config server '$hostPort'")) - - private def splitFirst(string: String, separator: Character): (String, Option[String]) = { - val (beginning, endWithSeparator) = string span { _ != separator } - (beginning, tailOption(endWithSeparator)) - } - - def tailOption(s: String) = { - if (s.isEmpty) None - else Some(s.tail) - } - - implicit def toJavaOptional[U <% V, V](option: Option[U]): Optional[V] = option match { - case Some(u) => Optional.of(u: V) - case None => Optional.empty() - } -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala index 4764f4698f0..2aab88d8319 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala @@ -6,13 +6,13 @@ package com.yahoo.container.standalone * TODO: copied from standalone-container. Move to separate lib module instead. */ object Environment { - def optionalYinstVariable(name: String) = { + def optionalInstallVariable(name: String) = { env(name.replace(".", "__")). orElse(systemProperty(name)) //for unit testing } - def yinstVariable(name: String) = { - optionalYinstVariable(name). + def installVariable(name: String) = { + optionalInstallVariable(name). getOrElse { throw new IllegalStateException("Environment variable not set: " + name) } diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala index 1f3ff652224..3ebea07bfe0 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala @@ -40,7 +40,7 @@ class StandaloneContainerApplication @Inject()(injector: Injector) extends Appli ConfiguredApplication.ensureVespaLoggingInitialized() - val applicationPath: Path = injectedApplicationPath.getOrElse(yinstApplicationPath) + val applicationPath: Path = injectedApplicationPath.getOrElse(installApplicationPath) val distributedFiles = new LocalFileDb(applicationPath) @@ -73,7 +73,7 @@ class StandaloneContainerApplication @Inject()(injector: Injector) extends Appli injector.getInstance(Key.get(classOf[Path], applicationPathName)) }.toOption - def yinstApplicationPath = path(yinstVariable(applicationLocationYinstVariable)) + def installApplicationPath = path(installVariable(applicationLocationInstallVariable)) override def start() { try { @@ -97,10 +97,10 @@ class StandaloneContainerApplication @Inject()(injector: Injector) extends Appli object StandaloneContainerApplication { val packageName = "standalone_jdisc_container" - val applicationLocationYinstVariable = s"$packageName.app_location" - val deploymentProfileYinstVariable = s"$packageName.deployment_profile" + val applicationLocationInstallVariable = s"$packageName.app_location" + val deploymentProfileInstallVariable = s"$packageName.deployment_profile" - val applicationPathName = Names.named(applicationLocationYinstVariable) + val applicationPathName = Names.named(applicationLocationInstallVariable) val disableNetworkingAnnotation = "JDisc.disableNetworking" val configModelRepoName = Names.named("ConfigModelRepo") @@ -143,9 +143,9 @@ object StandaloneContainerApplication { } def newContainerModelBuilder(networkingOption: Networking): ContainerModelBuilder = { - optionalYinstVariable(deploymentProfileYinstVariable) match { + optionalInstallVariable(deploymentProfileInstallVariable) match { case None => new ContainerModelBuilder(true, networkingOption) - case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigYinstVariables) + case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables) case profileName => throw new RuntimeException(s"Invalid deployment profile '$profileName'") } } diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala index 585221a8795..efa3edb7b7e 100644 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala @@ -11,8 +11,8 @@ import org.hamcrest.Matchers.{arrayContaining} * @author tonytv * @since 5. */ -class CloudConfigYinstVariablesTest { - def convert = CloudConfigYinstVariables.configServerConverter.convert _ +class CloudConfigInstallVariablesTest { + def convert = CloudConfigInstallVariables.configServerConverter.convert _ @Test def test_configserver_parsing { diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala index b70eefe0681..33f9a2e8594 100644 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala @@ -23,7 +23,7 @@ object StandaloneContainer { def withStandaloneContainer[T](containerNode: Node) { withTempDirectory { applicationDirectory => - System.setProperty(StandaloneContainerApplication.applicationLocationYinstVariable, applicationDirectory.toString) + System.setProperty(StandaloneContainerApplication.applicationLocationInstallVariable, applicationDirectory.toString) createServicesXml(applicationDirectory, containerNode) val driver = TestDriver.newInjectedApplicationInstance(classOf[StandaloneContainerApplication]) diff --git a/storage/src/tests/bucketdb/initializertest.cpp b/storage/src/tests/bucketdb/initializertest.cpp index cb76876c24e..2141dbf4b53 100644 --- a/storage/src/tests/bucketdb/initializertest.cpp +++ b/storage/src/tests/bucketdb/initializertest.cpp @@ -17,10 +17,13 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/bucketdb/lockablemap.hpp> #include <vespa/vdstestlib/cppunit/dirconfig.hpp> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/log/log.h> LOG_SETUP(".test.bucketdb.initializing"); +using document::FixedBucketSpaces; + namespace storage { typedef uint16_t PartitionId; @@ -444,10 +447,12 @@ struct FakePersistenceLayer : public StorageLink { << " for which we should not get a request"; fatal(ost.str()); } else { - for (DiskData::const_iterator it2 = it->second.begin(); - it2 != it->second.end(); ++it2) - { - reply->getBuckets().push_back(it2->first); + if (cmd.getBucket().getBucketSpace() == FixedBucketSpaces::default_space()) { + for (DiskData::const_iterator it2 = it->second.begin(); + it2 != it->second.end(); ++it2) + { + reply->getBuckets().push_back(it2->first); + } } } if (!fatalError.empty()) { diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp index b79894aee9a..77b661c2606 100644 --- a/storage/src/tests/distributor/bucketdbupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -13,18 +13,45 @@ #include <tests/distributor/distributortestutil.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/document/test/make_bucket_space.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/storage/distributor/simpleclusterinformation.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/vespalib/text/stringtokenizer.h> +#include <sstream> using namespace storage::api; using namespace storage::lib; using document::test::makeDocumentBucket; using document::test::makeBucketSpace; +using document::BucketSpace; +using document::FixedBucketSpaces; namespace storage::distributor { +namespace { + +std::string +getStringList(std::string s, uint32_t count) +{ + std::ostringstream ost; + for (uint32_t i = 0; i < count; ++i) { + if (i > 0) { + ost << ","; + } + ost << s; + } + return ost.str(); +} + +std::string +getRequestBucketInfoStrings(uint32_t count) +{ + return getStringList("Request bucket info", count); +} + +} + class BucketDBUpdaterTest : public CppUnit::TestFixture, public DistributorTestUtil { @@ -87,6 +114,9 @@ class BucketDBUpdaterTest : public CppUnit::TestFixture, CPPUNIT_TEST(batch_update_from_distributor_change_does_not_mark_diverging_replicas_as_trusted); CPPUNIT_TEST_SUITE_END(); +public: + BucketDBUpdaterTest(); + protected: void testNormalUsage(); void testDistributorChange(); @@ -161,10 +191,24 @@ protected: return clusterInfo; } + static std::string getNodeList(std::vector<uint16_t> nodes, size_t count); + + std::string getNodeList(std::vector<uint16_t> nodes); + + std::vector<uint16_t> + expandNodeVec(const std::vector<uint16_t> &nodes); + + std::vector<document::BucketSpace> _bucketSpaces; + + size_t messageCount(size_t messagesPerBucketSpace) const { + return messagesPerBucketSpace * _bucketSpaces.size(); + } + public: using OutdatedNodesMap = dbtransition::OutdatedNodesMap; void setUp() override { createLinks(); + _bucketSpaces = getBucketSpaces(); }; void tearDown() override { @@ -173,17 +217,17 @@ public: std::shared_ptr<RequestBucketInfoReply> getFakeBucketReply( const lib::ClusterState& state, - RequestBucketInfoCommand& cmd, + const RequestBucketInfoCommand& cmd, int storageIndex, - int bucketCount, - int invalidBucketCount = 0) + uint32_t bucketCount, + uint32_t invalidBucketCount = 0) { RequestBucketInfoReply* sreply = new RequestBucketInfoReply(cmd); sreply->setAddress(storageAddress(storageIndex)); api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); - for (int i=0; i<bucketCount + invalidBucketCount; i++) { + for (uint32_t i=0; i<bucketCount + invalidBucketCount; i++) { if (!getBucketDBUpdater().getDistributorComponent() .ownsBucketInState(state, makeDocumentBucket(document::BucketId(16, i)))) { continue; @@ -214,17 +258,17 @@ public: return std::shared_ptr<api::RequestBucketInfoReply>(sreply); } - void fakeBucketReply( - const lib::ClusterState& state, - RequestBucketInfoCommand& cmd, - int storageIndex, - int bucketCount, - int invalidBucketCount = 0) + void fakeBucketReply(const lib::ClusterState &state, + const api::StorageCommand &cmd, + uint32_t bucketCount, + uint32_t invalidBucketCount = 0) { + CPPUNIT_ASSERT(cmd.getType() == MessageType::REQUESTBUCKETINFO); + const api::StorageMessageAddress &address(*cmd.getAddress()); getBucketDBUpdater().onRequestBucketInfoReply( getFakeBucketReply(state, - cmd, - storageIndex, + dynamic_cast<const RequestBucketInfoCommand &>(cmd), + address.getIndex(), bucketCount, invalidBucketCount)); } @@ -332,22 +376,15 @@ public: } void completeBucketInfoGathering(const lib::ClusterState& state, - uint32_t expectedMsgs, - uint32_t nBuckets = 1) + size_t expectedMsgs, + uint32_t bucketCount = 1, + uint32_t invalidBucketCount = 0) { - CPPUNIT_ASSERT_EQUAL(size_t(expectedMsgs), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(expectedMsgs, _sender.commands.size()); for (uint32_t i = 0; i < _sender.commands.size(); i++) { - CPPUNIT_ASSERT(_sender.commands[i]->getType() == - MessageType::REQUESTBUCKETINFO); - - const api::StorageMessageAddress& address( - *_sender.commands[i]->getAddress()); - fakeBucketReply( - state, - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - address.getIndex(), - nBuckets); + fakeBucketReply(state, *_sender.commands[i], + bucketCount, invalidBucketCount); } } @@ -383,12 +420,12 @@ public: setSystemState(newState); - for (uint32_t i=0; i<numStorageNodes; i++) { + for (uint32_t i=0; i< messageCount(numStorageNodes); i++) { CPPUNIT_ASSERT(_sender.commands[i]->getType() == MessageType::REQUESTBUCKETINFO); const api::StorageMessageAddress *address = _sender.commands[i]->getAddress(); - CPPUNIT_ASSERT_EQUAL(i, (uint32_t)address->getIndex()); + CPPUNIT_ASSERT_EQUAL((uint32_t)(i / _bucketSpaces.size()), (uint32_t)address->getIndex()); } } @@ -401,11 +438,8 @@ public: "distributor:1 storage:%d", numStorageNodes)); lib::ClusterState newState(state); - for (uint32_t i=0; i<numStorageNodes; i++) { - fakeBucketReply(newState, - *((RequestBucketInfoCommand*)_sender.commands[i].get()), - i, - numBuckets); + for (uint32_t i=0; i< messageCount(numStorageNodes); i++) { + fakeBucketReply(newState, *_sender.commands[i], numBuckets); } assertCorrectBuckets(numBuckets, state); } @@ -580,12 +614,19 @@ public: CPPUNIT_TEST_SUITE_REGISTRATION(BucketDBUpdaterTest); +BucketDBUpdaterTest::BucketDBUpdaterTest() + : CppUnit::TestFixture(), + DistributorTestUtil(), + _bucketSpaces() +{ +} + void BucketDBUpdaterTest::testNormalUsage() { setSystemState(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); - CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(3), _sender.commands.size()); // Ensure distribution hash is set correctly CPPUNIT_ASSERT_EQUAL( @@ -594,10 +635,8 @@ BucketDBUpdaterTest::testNormalUsage() dynamic_cast<const RequestBucketInfoCommand&>( *_sender.commands[0]).getDistributionHash()); - fakeBucketReply( - lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0]), - 0, 10); + fakeBucketReply(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + *_sender.commands[0], 10); _sender.clear(); @@ -605,17 +644,12 @@ BucketDBUpdaterTest::testNormalUsage() // change is only implemented after completion of previous cluster state setSystemState(lib::ClusterState("distributor:2 .0.s:i storage:3")); - CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(3), _sender.commands.size()); // Expect reply of first set SystemState request. CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.replies.size()); - for (uint32_t i = 0; i < 3; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, 10); - } - + completeBucketInfoGathering(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + messageCount(3), 10); assertCorrectBuckets(10, "distributor:2 storage:3"); } @@ -626,13 +660,9 @@ BucketDBUpdaterTest::testDistributorChange() // First sends request setSystemState(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); - CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); - for (uint32_t i = 0; i < 3; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, numBuckets); - } + CPPUNIT_ASSERT_EQUAL(messageCount(3), _sender.commands.size()); + completeBucketInfoGathering(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + messageCount(3), numBuckets); _sender.clear(); // No change from initializing to up (when done with last job) @@ -648,13 +678,9 @@ BucketDBUpdaterTest::testDistributorChange() // Removing distributor. Need to refetch new data from all nodes. setSystemState(lib::ClusterState("distributor:2 storage:3")); - CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); - for (uint32_t i = 0; i < 3; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:2 storage:3"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, numBuckets); - } + CPPUNIT_ASSERT_EQUAL(messageCount(3), _sender.commands.size()); + completeBucketInfoGathering(lib::ClusterState("distributor:2 storage:3"), + messageCount(3), numBuckets); _sender.clear(); assertCorrectBuckets(numBuckets, "distributor:2 storage:3"); } @@ -667,13 +693,9 @@ BucketDBUpdaterTest::testDistributorChangeWithGrouping() int numBuckets = 100; setSystemState(lib::ClusterState("distributor:6 storage:6")); - CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); - for (uint32_t i = 0; i < 6; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:6 storage:6"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, numBuckets); - } + CPPUNIT_ASSERT_EQUAL(messageCount(6), _sender.commands.size()); + completeBucketInfoGathering(lib::ClusterState("distributor:6 storage:6"), + messageCount(6), numBuckets); _sender.clear(); // Distributor going down in other group, no change @@ -693,7 +715,7 @@ BucketDBUpdaterTest::testDistributorChangeWithGrouping() // Changed grouping cause change setDistribution(getDistConfig6Nodes4Groups()); - CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(6), _sender.commands.size()); } void @@ -701,16 +723,13 @@ BucketDBUpdaterTest::testNormalUsageInitializing() { setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1 .0.s:i")); - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); // Not yet passing on system state. CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); - fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.commands[0].get()), - 0, - 10, - 10); + completeBucketInfoGathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), + _bucketSpaces.size(), 10, 10); assertCorrectBuckets(10, "distributor:1 storage:1"); @@ -727,12 +746,10 @@ BucketDBUpdaterTest::testNormalUsageInitializing() setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1")); // Send a new request bucket info up. - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); - fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.commands[0].get()), - 0, - 20); + completeBucketInfoGathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), + _bucketSpaces.size(), 20); // Pass on cluster state and recheck buckets now. CPPUNIT_ASSERT_EQUAL(size_t(1), _senderDown.commands.size()); @@ -746,33 +763,34 @@ BucketDBUpdaterTest::testFailedRequestBucketInfo() setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1")); // 2 messages sent up: 1 to the nodes, and one reply to the setsystemstate. - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); { - std::shared_ptr<api::RequestBucketInfoReply> reply = - getFakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.commands[0].get()), - 0, - 10); + for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { + std::shared_ptr<api::RequestBucketInfoReply> reply = + getFakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), + *((RequestBucketInfoCommand*)_sender.commands[i].get()), + 0, + 10); + reply->setResult(api::ReturnCode::NOT_CONNECTED); + getBucketDBUpdater().onRequestBucketInfoReply(reply); + } - reply->setResult(api::ReturnCode::NOT_CONNECTED); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - // Trigger that delayed message is sent + // Trigger that delayed message is sent getClock().addSecondsToTime(10); getBucketDBUpdater().resendDelayedMessages(); } // Should be resent. - CPPUNIT_ASSERT_EQUAL(std::string("Request bucket info," - "Request bucket info"), + CPPUNIT_ASSERT_EQUAL(getRequestBucketInfoStrings(messageCount(2)), _sender.getCommands()); CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); - fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.commands[1].get()), - 0, - 10); + for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { + fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), + *_sender.commands[_bucketSpaces.size() + i], 10); + } for (int i=0; i<10; i++) { CPPUNIT_ASSERT_EQUAL( @@ -791,14 +809,14 @@ BucketDBUpdaterTest::testEncodeErrorHandling() { setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1")); - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); // Not yet passing on system state. CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); - { + for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { std::shared_ptr<api::RequestBucketInfoReply> reply = getFakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.commands[0].get()), + *((RequestBucketInfoCommand*)_sender.commands[i].get()), 0, 10); @@ -815,21 +833,15 @@ BucketDBUpdaterTest::testDownWhileInit() setStorageNodes(3); fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), - *((RequestBucketInfoCommand*)_sender.commands[0].get()), - 0, - 5); + *_sender.commands[0], 5); setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:d")); fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), - *((RequestBucketInfoCommand*)_sender.commands[2].get()), - 2, - 5); + *_sender.commands[2], 5); fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), - *((RequestBucketInfoCommand*)_sender.commands[1].get()), - 1, - 5); + *_sender.commands[1], 5); } bool @@ -844,6 +856,42 @@ BucketDBUpdaterTest::bucketExistsThatHasNode(int bucketCount, uint16_t node) con return false; } +std::string +BucketDBUpdaterTest::getNodeList(std::vector<uint16_t> nodes, size_t count) +{ + std::ostringstream ost; + bool first = true; + for (const auto &node : nodes) { + for (uint32_t i = 0; i < count; ++i) { + if (!first) { + ost << ","; + } + ost << node; + first = false; + } + } + return ost.str(); +} + +std::string +BucketDBUpdaterTest::getNodeList(std::vector<uint16_t> nodes) +{ + return getNodeList(std::move(nodes), _bucketSpaces.size()); +} + +std::vector<uint16_t> +BucketDBUpdaterTest::expandNodeVec(const std::vector<uint16_t> &nodes) +{ + std::vector<uint16_t> res; + size_t count = _bucketSpaces.size(); + for (const auto &node : nodes) { + for (uint32_t i = 0; i < count; ++i) { + res.push_back(node); + } + } + return res; +} + void BucketDBUpdaterTest::testNodeDown() { @@ -903,16 +951,13 @@ BucketDBUpdaterTest::testInitializingWhileRecheck() lib::ClusterState systemState("distributor:1 storage:2 .0.s:i .0.i:0.1"); setSystemState(systemState); - CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(2), _sender.commands.size()); CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); getBucketDBUpdater().recheckBucketInfo(1, makeDocumentBucket(document::BucketId(16, 3))); - for (int i=0; i<2; i++) { - fakeBucketReply(systemState, - *((RequestBucketInfoCommand*)_sender.commands[i].get()), - i, - 100); + for (uint32_t i = 0; i < messageCount(2); ++i) { + fakeBucketReply(systemState, *_sender.commands[i], 100); } // Now we can pass on system state. @@ -931,35 +976,36 @@ BucketDBUpdaterTest::testBitChange() { setSystemState(lib::ClusterState("bits:14 storage:1 distributor:2")); - CPPUNIT_ASSERT_EQUAL(1, (int)_sender.commands.size()); - - CPPUNIT_ASSERT(_sender.commands[0]->getType() == MessageType::REQUESTBUCKETINFO); - - RequestBucketInfoReply* sreply = - new RequestBucketInfoReply(*((RequestBucketInfoCommand*)_sender.commands[0].get())); - sreply->setAddress(storageAddress(0)); - api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); - + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); + + for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) { + CPPUNIT_ASSERT(_sender.commands[bsi]->getType() == MessageType::REQUESTBUCKETINFO); + const auto &req = dynamic_cast<const RequestBucketInfoCommand &>(*_sender.commands[bsi]); + RequestBucketInfoReply* sreply = new RequestBucketInfoReply(req); + sreply->setAddress(storageAddress(0)); + api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); + if (req.getBucketSpace() == FixedBucketSpaces::default_space()) { + int cnt=0; + for (int i=0; cnt < 2; i++) { + lib::Distribution distribution = defaultDistributorBucketSpace().getDistribution(); + std::vector<uint16_t> distributors; + if (distribution.getIdealDistributorNode( + lib::ClusterState("redundancy:1 bits:14 storage:1 distributor:2"), + document::BucketId(16, i)) + == 0) + { + vec.push_back(api::RequestBucketInfoReply::Entry( + document::BucketId(16, i), + api::BucketInfo(10,1,1))); - int cnt=0; - for (int i=0; cnt < 2; i++) { - lib::Distribution distribution = defaultDistributorBucketSpace().getDistribution(); - std::vector<uint16_t> distributors; - if (distribution.getIdealDistributorNode( - lib::ClusterState("redundancy:1 bits:14 storage:1 distributor:2"), - document::BucketId(16, i)) - == 0) - { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(10,1,1))); - - bucketlist.push_back(document::BucketId(16, i)); - cnt++; + bucketlist.push_back(document::BucketId(16, i)); + cnt++; + } + } } - } - getBucketDBUpdater().onRequestBucketInfoReply(std::shared_ptr<RequestBucketInfoReply>(sreply)); + getBucketDBUpdater().onRequestBucketInfoReply(std::shared_ptr<RequestBucketInfoReply>(sreply)); + } } CPPUNIT_ASSERT_EQUAL( @@ -975,29 +1021,31 @@ BucketDBUpdaterTest::testBitChange() _sender.clear(); setSystemState(lib::ClusterState("bits:16 storage:1 distributor:2")); - CPPUNIT_ASSERT_EQUAL(1, (int)_sender.commands.size()); - - CPPUNIT_ASSERT(_sender.commands[0]->getType() == MessageType::REQUESTBUCKETINFO); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); + for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) { + + CPPUNIT_ASSERT(_sender.commands[bsi]->getType() == MessageType::REQUESTBUCKETINFO); + const auto &req = dynamic_cast<const RequestBucketInfoCommand &>(*_sender.commands[bsi]); + RequestBucketInfoReply* sreply = new RequestBucketInfoReply(req); + sreply->setAddress(storageAddress(0)); + sreply->setResult(api::ReturnCode::OK); + if (req.getBucketSpace() == FixedBucketSpaces::default_space()) { + api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); + + for (uint32_t i = 0; i < 3; ++i) { + vec.push_back(api::RequestBucketInfoReply::Entry( + document::BucketId(16, i), + api::BucketInfo(10,1,1))); + } - RequestBucketInfoReply* sreply = - new RequestBucketInfoReply( - *((RequestBucketInfoCommand*)_sender.commands[0].get())); - sreply->setAddress(storageAddress(0)); - sreply->setResult(api::ReturnCode::OK); - api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); + vec.push_back(api::RequestBucketInfoReply::Entry( + document::BucketId(16, 4), + api::BucketInfo(10,1,1))); + } - for (uint32_t i = 0; i < 3; ++i) { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(10,1,1))); + getBucketDBUpdater().onRequestBucketInfoReply( + std::shared_ptr<RequestBucketInfoReply>(sreply)); } - - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, 4), - api::BucketInfo(10,1,1))); - - getBucketDBUpdater().onRequestBucketInfoReply( - std::shared_ptr<RequestBucketInfoReply>(sreply)); } CPPUNIT_ASSERT_EQUAL( @@ -1256,7 +1304,7 @@ void BucketDBUpdaterTest::testNotifyChangeWithPendingStateQueuesBucketInfoRequests() { setSystemState(lib::ClusterState("distributor:1 storage:1")); - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); { api::BucketInfo info(8999, 300, 3000, 500, 5000, false, false); @@ -1266,18 +1314,16 @@ BucketDBUpdaterTest::testNotifyChangeWithPendingStateQueuesBucketInfoRequests() getBucketDBUpdater().onNotifyBucketChange(cmd); } - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); - fakeBucketReply( - lib::ClusterState("distributor:1 storage:1"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0]), - 0, 10); + completeBucketInfoGathering(lib::ClusterState("distributor:1 storage:1"), + _bucketSpaces.size(), 10); - CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size() + 1, _sender.commands.size()); { api::RequestBucketInfoCommand& rbi( - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[1])); + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[_bucketSpaces.size()])); CPPUNIT_ASSERT_EQUAL(size_t(1), rbi.getBuckets().size()); CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1), rbi.getBuckets()[0]); } @@ -1286,10 +1332,10 @@ BucketDBUpdaterTest::testNotifyChangeWithPendingStateQueuesBucketInfoRequests() // Queue must be cleared once pending state is enabled. { lib::ClusterState state("distributor:1 storage:2"); - uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; setAndEnableClusterState(state, expectedMsgs, dummyBucketsToReturn); } - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); { api::RequestBucketInfoCommand& rbi( dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0])); @@ -1527,32 +1573,32 @@ void BucketDBUpdaterTest::testPendingClusterStateSendMessages() { CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2"), + getNodeList({0, 1, 2}), getSentNodes("cluster:d", "distributor:1 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1"), + getNodeList({0, 1}), getSentNodes("cluster:d", "distributor:1 storage:3 .2.s:m")); CPPUNIT_ASSERT_EQUAL( - std::string("2"), + getNodeList({2}), getSentNodes("distributor:1 storage:2", "distributor:1 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("2,3,4,5"), + getNodeList({2, 3, 4, 5}), getSentNodes("distributor:1 storage:2", "distributor:1 storage:6")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2"), + getNodeList({0, 1, 2}), getSentNodes("distributor:4 storage:3", "distributor:3 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2,3"), + getNodeList({0, 1, 2, 3}), getSentNodes("distributor:4 storage:3", "distributor:4 .2.s:d storage:4")); @@ -1567,17 +1613,17 @@ BucketDBUpdaterTest::testPendingClusterStateSendMessages() "distributor:4 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("2"), + getNodeList({2}), getSentNodes("distributor:3 storage:3 .2.s:i", "distributor:3 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("1"), + getNodeList({1}), getSentNodes("distributor:3 storage:3 .1.s:d", "distributor:3 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("1,2,4"), + getNodeList({1, 2, 4}), getSentNodes("distributor:3 storage:4 .1.s:d .2.s:i", "distributor:3 storage:5")); @@ -1602,7 +1648,7 @@ BucketDBUpdaterTest::testPendingClusterStateSendMessages() "distributor:3 .2.s:m storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2"), + getNodeList({0, 1, 2}), getSentNodes("distributor:3 .2.s:m storage:3", "distributor:3 .2.s:d storage:3")); @@ -1612,21 +1658,21 @@ BucketDBUpdaterTest::testPendingClusterStateSendMessages() "distributor:3 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2"), + getNodeList({0, 1, 2}), getSentNodesDistributionChanged("distributor:3 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1"), + getNodeList({0, 1}), getSentNodes("distributor:10 storage:2", "distributor:10 .1.s:d storage:2")); CPPUNIT_ASSERT_EQUAL( - std::string("1"), + getNodeList({1}), getSentNodes("distributor:2 storage:2", "distributor:2 storage:2 .1.d:3 .1.d.1.s:d")); CPPUNIT_ASSERT_EQUAL( - std::string("1"), + getNodeList({1}), getSentNodes("distributor:2 storage:2 .1.s:d", "distributor:2 storage:2 .1.d:3 .1.d.1.s:d")); @@ -1636,7 +1682,7 @@ BucketDBUpdaterTest::testPendingClusterStateSendMessages() "distributor:3 .2.s:i storage:2")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2"), + getNodeList({0, 1, 2}), getSentNodes("distributor:3 storage:3", "distributor:3 .2.s:s storage:3")); @@ -1646,7 +1692,7 @@ BucketDBUpdaterTest::testPendingClusterStateSendMessages() "distributor:3 .2.s:d storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("1"), + getNodeList({1}), getSentNodes("distributor:3 storage:3 .1.s:m", "distributor:3 storage:3")); @@ -1672,7 +1718,7 @@ BucketDBUpdaterTest::testPendingClusterStateReceive() clock, clusterInfo, sender, getBucketSpaceRepo(), cmd, outdatedNodesMap, api::Timestamp(1))); - CPPUNIT_ASSERT_EQUAL(3, (int)sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(3), sender.commands.size()); sortSentMessagesByIndex(sender); @@ -1713,7 +1759,7 @@ BucketDBUpdaterTest::testPendingClusterStateWithGroupDown() // Entire group 1 goes down. Must refetch from all nodes. CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2,3,4,5"), + getNodeList({0, 1, 2, 3, 4, 5}), getSentNodes("distributor:6 storage:6", "distributor:6 .2.s:d .3.s:d storage:6")); @@ -1733,7 +1779,7 @@ BucketDBUpdaterTest::testPendingClusterStateWithGroupDownAndNoHandover() // Group is down, but config says to not do anything about it. CPPUNIT_ASSERT_EQUAL( - std::string(""), + getNodeList({0, 1, 2, 3, 4, 5}, _bucketSpaces.size() - 1), getSentNodes("distributor:6 storage:6", "distributor:6 .2.s:d .3.s:d storage:6")); } @@ -1973,7 +2019,7 @@ BucketDBUpdaterTest::testNoDbResurrectionForBucketNotOwnedInCurrentState() document::BucketId bucket(16, 3); lib::ClusterState stateBefore("distributor:1 storage:1"); { - uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); } _sender.clear(); @@ -1988,7 +2034,7 @@ BucketDBUpdaterTest::testNoDbResurrectionForBucketNotOwnedInCurrentState() lib::ClusterState stateAfter("distributor:3 storage:3"); { - uint32_t expectedMsgs = 2, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = messageCount(2), dummyBucketsToReturn = 1; setAndEnableClusterState(stateAfter, expectedMsgs, dummyBucketsToReturn); } CPPUNIT_ASSERT(!getBucketDBUpdater().getDistributorComponent() @@ -2005,7 +2051,7 @@ BucketDBUpdaterTest::testNoDbResurrectionForBucketNotOwnedInPendingState() document::BucketId bucket(16, 3); lib::ClusterState stateBefore("distributor:1 storage:1"); { - uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); } _sender.clear(); @@ -2042,38 +2088,35 @@ BucketDBUpdaterTest::testClusterStateAlwaysSendsFullFetchWhenDistributionChangeP { lib::ClusterState stateBefore("distributor:6 storage:6"); { - uint32_t expectedMsgs = 6, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = messageCount(6), dummyBucketsToReturn = 1; setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); } _sender.clear(); std::string distConfig(getDistConfig6Nodes3Groups()); setDistribution(distConfig); sortSentMessagesByIndex(_sender); - CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(6), _sender.commands.size()); // Suddenly, a wild cluster state change appears! Even though this state // does not in itself imply any bucket changes, it will still overwrite the // pending cluster state and thus its state of pending bucket info requests. setSystemState(lib::ClusterState("distributor:6 .2.t:12345 storage:6")); - CPPUNIT_ASSERT_EQUAL(size_t(12), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(12), _sender.commands.size()); - // Send replies for first 6 (outdated requests). + // Send replies for first messageCount(6) (outdated requests). int numBuckets = 10; - for (uint32_t i = 0; i < 6; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:6 storage:6"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, numBuckets); + for (uint32_t i = 0; i < messageCount(6); ++i) { + fakeBucketReply(lib::ClusterState("distributor:6 storage:6"), + *_sender.commands[i], numBuckets); } // No change from these. assertCorrectBuckets(1, "distributor:6 storage:6"); // Send for current pending. - for (uint32_t i = 0; i < 6; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:6 .2.t:12345 storage:6"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i + 6]), - i, numBuckets); + for (uint32_t i = 0; i < messageCount(6); ++i) { + fakeBucketReply(lib::ClusterState("distributor:6 .2.t:12345 storage:6"), + *_sender.commands[i + messageCount(6)], + numBuckets); } assertCorrectBuckets(numBuckets, "distributor:6 storage:6"); _sender.clear(); @@ -2086,7 +2129,7 @@ BucketDBUpdaterTest::testClusterStateAlwaysSendsFullFetchWhenDistributionChangeP void BucketDBUpdaterTest::testChangedDistributionConfigTriggersRecoveryMode() { - setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), 6, 20); + setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), messageCount(6), 20); _sender.clear(); // First cluster state; implicit scan of all buckets which does not // use normal recovery mode ticking-path. @@ -2098,13 +2141,11 @@ BucketDBUpdaterTest::testChangedDistributionConfigTriggersRecoveryMode() // No replies received yet, still no recovery mode. CPPUNIT_ASSERT(!_distributor->isInRecoveryMode()); - CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(6), _sender.commands.size()); uint32_t numBuckets = 10; - for (uint32_t i = 0; i < 6; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:6 storage:6"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, numBuckets); + for (uint32_t i = 0; i < messageCount(6); ++i) { + fakeBucketReply(lib::ClusterState("distributor:6 storage:6"), + *_sender.commands[i], numBuckets); } // Pending cluster state (i.e. distribution) has been enabled, which should @@ -2118,7 +2159,7 @@ BucketDBUpdaterTest::testNewlyAddedBucketsHaveCurrentTimeAsGcTimestamp() getClock().setAbsoluteTimeInSeconds(101234); lib::ClusterState stateBefore("distributor:1 storage:1"); { - uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); } @@ -2134,7 +2175,7 @@ BucketDBUpdaterTest::testNewerMutationsNotOverwrittenByEarlierBucketFetch() { { lib::ClusterState stateBefore("distributor:1 storage:1 .0.s:i"); - uint32_t expectedMsgs = 1, dummyBucketsToReturn = 0; + uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 0; // This step is required to make the distributor ready for accepting // the below explicit database insertion towards node 0. setAndEnableClusterState(stateBefore, expectedMsgs, @@ -2144,7 +2185,7 @@ BucketDBUpdaterTest::testNewerMutationsNotOverwrittenByEarlierBucketFetch() getClock().setAbsoluteTimeInSeconds(1000); lib::ClusterState state("distributor:1 storage:1"); setSystemState(state); - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); // Before replying with the bucket info, simulate the arrival of a mutation // reply that alters the state of the bucket with information that will be @@ -2168,11 +2209,9 @@ BucketDBUpdaterTest::testNewerMutationsNotOverwrittenByEarlierBucketFetch() // happening before t=1000 but receiving a reply at t>1000 does not affect // correctness, as this should contain the same bucket info as that // contained in the full bucket reply and the DB update is thus idempotent. - fakeBucketReply( - state, - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0]), - 0, - bucketsReturned); + for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { + fakeBucketReply(state, *_sender.commands[i], bucketsReturned); + } BucketDatabase::Entry e(getBucket(bucket)); CPPUNIT_ASSERT_EQUAL(uint32_t(1), e->getNodeCount()); @@ -2231,8 +2270,9 @@ void BucketDBUpdaterTest::preemptedDistrChangeCarriesNodeSetOverToNextStateFetch() { CPPUNIT_ASSERT_EQUAL( - (nodeVec{0, 1, 2, 3, 4, 5}), - getSentNodesWithPreemption("version:1 distributor:6 storage:6", 6, + expandNodeVec({0, 1, 2, 3, 4, 5}), + getSentNodesWithPreemption("version:1 distributor:6 storage:6", + messageCount(6), "version:2 distributor:6 .5.s:d storage:6", "version:3 distributor:6 storage:6")); } @@ -2241,9 +2281,10 @@ void BucketDBUpdaterTest::preemptedStorChangeCarriesNodeSetOverToNextStateFetch() { CPPUNIT_ASSERT_EQUAL( - (nodeVec{2, 3}), + expandNodeVec({2, 3}), getSentNodesWithPreemption( - "version:1 distributor:6 storage:6 .2.s:d", 5, + "version:1 distributor:6 storage:6 .2.s:d", + messageCount(5), "version:2 distributor:6 storage:6 .2.s:d .3.s:d", "version:3 distributor:6 storage:6")); } @@ -2252,9 +2293,10 @@ void BucketDBUpdaterTest::preemptedStorageNodeDownMustBeReFetched() { CPPUNIT_ASSERT_EQUAL( - (nodeVec{2}), + expandNodeVec({2}), getSentNodesWithPreemption( - "version:1 distributor:6 storage:6", 6, + "version:1 distributor:6 storage:6", + messageCount(6), "version:2 distributor:6 storage:6 .2.s:d", "version:3 distributor:6 storage:6")); } @@ -2265,7 +2307,8 @@ BucketDBUpdaterTest::doNotSendToPreemptedNodeNowInDownState() CPPUNIT_ASSERT_EQUAL( nodeVec{}, getSentNodesWithPreemption( - "version:1 distributor:6 storage:6 .2.s:d", 5, + "version:1 distributor:6 storage:6 .2.s:d", + messageCount(5), "version:2 distributor:6 storage:6", // Sends to 2. "version:3 distributor:6 storage:6 .2.s:d")); // 2 down again. } @@ -2276,9 +2319,10 @@ BucketDBUpdaterTest::doNotSendToPreemptedNodeNotPartOfNewState() // Even though 100 nodes are preempted, not all of these should be part // of the request afterwards when only 6 are part of the state. CPPUNIT_ASSERT_EQUAL( - (nodeVec{0, 1, 2, 3, 4, 5}), + expandNodeVec({0, 1, 2, 3, 4, 5}), getSentNodesWithPreemption( - "version:1 distributor:6 storage:100", 100, + "version:1 distributor:6 storage:100", + messageCount(100), "version:2 distributor:5 .4.s:d storage:100", "version:3 distributor:6 storage:6")); } @@ -2288,7 +2332,7 @@ BucketDBUpdaterTest::outdatedNodeSetClearedAfterSuccessfulStateCompletion() { lib::ClusterState stateBefore( "version:1 distributor:6 storage:6 .1.t:1234"); - uint32_t expectedMsgs = 6, dummyBucketsToReturn = 10; + uint32_t expectedMsgs = messageCount(6), dummyBucketsToReturn = 10; setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); _sender.clear(); // New cluster state that should not by itself trigger any new fetches, @@ -2327,7 +2371,7 @@ BucketDBUpdaterTest::changedDiskSetTriggersReFetch() { // Same number of online disks, but the set of disks has changed. CPPUNIT_ASSERT_EQUAL( - std::string("1"), + getNodeList({1}), getSentNodes("distributor:2 storage:2 .1.d:3 .1.d.2.s:d", "distributor:2 storage:2 .1.d:3 .1.d.1.s:d")); } @@ -2343,7 +2387,7 @@ BucketDBUpdaterTest::changedDiskSetTriggersReFetch() void BucketDBUpdaterTest::nodeMissingFromConfigIsTreatedAsNeedingOwnershipTransfer() { - uint32_t expectedMsgs = 3, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = messageCount(3), dummyBucketsToReturn = 1; setAndEnableClusterState(lib::ClusterState("distributor:3 storage:3"), expectedMsgs, dummyBucketsToReturn); _sender.clear(); @@ -2371,11 +2415,11 @@ BucketDBUpdaterTest::nodeMissingFromConfigIsTreatedAsNeedingOwnershipTransfer() // Attempt to apply state with {0, 1} set. This will compare the new state // with the previous state, which still has node 2. - expectedMsgs = 2; + expectedMsgs = messageCount(2); setAndEnableClusterState(lib::ClusterState("distributor:2 storage:2"), expectedMsgs, dummyBucketsToReturn); - CPPUNIT_ASSERT_EQUAL((nodeVec{0, 1}), getSendSet()); + CPPUNIT_ASSERT_EQUAL(expandNodeVec({0, 1}), getSendSet()); } void @@ -2413,7 +2457,7 @@ BucketDBUpdaterTest::changed_distribution_config_implies_ownership_transfer() void BucketDBUpdaterTest::transition_time_tracked_for_single_state_change() { - completeStateTransitionInSeconds("distributor:2 storage:2", 5, 2); + completeStateTransitionInSeconds("distributor:2 storage:2", 5, messageCount(2)); CPPUNIT_ASSERT_EQUAL(uint64_t(5000), lastTransitionTimeInMillis()); } @@ -2421,8 +2465,8 @@ BucketDBUpdaterTest::transition_time_tracked_for_single_state_change() void BucketDBUpdaterTest::transition_time_reset_across_non_preempting_state_changes() { - completeStateTransitionInSeconds("distributor:2 storage:2", 5, 2); - completeStateTransitionInSeconds("distributor:2 storage:3", 3, 1); + completeStateTransitionInSeconds("distributor:2 storage:2", 5, messageCount(2)); + completeStateTransitionInSeconds("distributor:2 storage:3", 3, messageCount(1)); CPPUNIT_ASSERT_EQUAL(uint64_t(3000), lastTransitionTimeInMillis()); } @@ -2431,13 +2475,13 @@ void BucketDBUpdaterTest::transition_time_tracked_for_distribution_config_change() { lib::ClusterState state("distributor:2 storage:2"); - setAndEnableClusterState(state, 2, 1); + setAndEnableClusterState(state, messageCount(2), 1); _sender.clear(); std::string distConfig(getDistConfig3Nodes1Group()); setDistribution(distConfig); getClock().addSecondsToTime(4); - completeBucketInfoGathering(state, 2); + completeBucketInfoGathering(state, messageCount(2)); CPPUNIT_ASSERT_EQUAL(uint64_t(4000), lastTransitionTimeInMillis()); } @@ -2451,7 +2495,7 @@ BucketDBUpdaterTest::transition_time_tracked_across_preempted_transitions() // Pre-empted with new state here, which will push out the old pending // state and replace it with a new one. We should still count the time // used processing the old state. - completeStateTransitionInSeconds("distributor:2 storage:3", 3, 3); + completeStateTransitionInSeconds("distributor:2 storage:3", 3, messageCount(3)); CPPUNIT_ASSERT_EQUAL(uint64_t(8000), lastTransitionTimeInMillis()); } diff --git a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp index 2617c843912..14df6ee5016 100644 --- a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp +++ b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp @@ -9,10 +9,11 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <tests/common/hostreporter/util.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/storage/distributor/bucket_spaces_stats_provider.h> -namespace storage { -namespace distributor { +namespace storage::distributor { +using PerNodeBucketSpacesStats = BucketSpacesStatsProvider::PerNodeBucketSpacesStats; using End = vespalib::JsonStream::End; using File = vespalib::File; using Object = vespalib::JsonStream::Object; @@ -24,17 +25,26 @@ class DistributorHostInfoReporterTest : public CppUnit::TestFixture CPPUNIT_TEST(hostInfoAllInfo); CPPUNIT_TEST(generateExampleJson); CPPUNIT_TEST(noReportGeneratedIfDisabled); + CPPUNIT_TEST(bucket_spaces_stats_are_reported); CPPUNIT_TEST_SUITE_END(); void hostInfoWithPutLatenciesOnly(); void hostInfoAllInfo(); - void verifyReportedNodeLatencies( - const vespalib::Slime& root, - uint16_t node, - int64_t latencySum, - int64_t count); + void verifyReportedNodeLatencies(const vespalib::Slime& root, + uint16_t nodeIndex, + int64_t latencySum, + int64_t count); + void verifyBucketSpaceStats(const vespalib::Slime& root, + uint16_t nodeIndex, + const vespalib::string& bucketSpaceName, + size_t bucketsTotal, + size_t bucketsPending); + void verifyBucketSpaceStats(const vespalib::Slime& root, + uint16_t nodeIndex, + const vespalib::string& bucketSpaceName); void generateExampleJson(); void noReportGeneratedIfDisabled(); + void bucket_spaces_stats_are_reported(); }; CPPUNIT_TEST_SUITE_REGISTRATION(DistributorHostInfoReporterTest); @@ -71,6 +81,14 @@ struct MockedMinReplicaProvider : MinReplicaProvider } }; +struct MockedBucketSpacesStatsProvider : public BucketSpacesStatsProvider { + PerNodeBucketSpacesStats stats; + + PerNodeBucketSpacesStats getBucketSpacesStats() const override { + return stats; + } +}; + const vespalib::slime::Inspector& getNode(const vespalib::Slime& root, uint16_t nodeIndex) { @@ -97,37 +115,80 @@ getLatenciesForNode(const vespalib::Slime& root, uint16_t nodeIndex) return getNode(root, nodeIndex)["ops-latency"]; } -} // anon ns +const vespalib::slime::Inspector& +getBucketSpaceStats(const vespalib::Slime& root, uint16_t nodeIndex, const vespalib::string& bucketSpaceName) +{ + const auto& bucketSpaces = getNode(root, nodeIndex)["bucket-spaces"]; + for (size_t i = 0; i < bucketSpaces.entries(); ++i) { + if (bucketSpaces[i]["name"].asString().make_stringref() == bucketSpaceName) { + return bucketSpaces[i]; + } + } + throw std::runtime_error("No bucket space found with name " + bucketSpaceName); +} + +} void -DistributorHostInfoReporterTest::verifyReportedNodeLatencies( - const vespalib::Slime& root, - uint16_t node, - int64_t latencySum, - int64_t count) +DistributorHostInfoReporterTest::verifyReportedNodeLatencies(const vespalib::Slime& root, + uint16_t nodeIndex, + int64_t latencySum, + int64_t count) { - auto& latencies = getLatenciesForNode(root, node); + auto& latencies = getLatenciesForNode(root, nodeIndex); CPPUNIT_ASSERT_EQUAL(latencySum, latencies["put"]["latency-ms-sum"].asLong()); CPPUNIT_ASSERT_EQUAL(count, latencies["put"]["count"].asLong()); } void -DistributorHostInfoReporterTest::hostInfoWithPutLatenciesOnly() +DistributorHostInfoReporterTest::verifyBucketSpaceStats(const vespalib::Slime& root, + uint16_t nodeIndex, + const vespalib::string& bucketSpaceName, + size_t bucketsTotal, + size_t bucketsPending) +{ + const auto &stats = getBucketSpaceStats(root, nodeIndex, bucketSpaceName); + CPPUNIT_ASSERT_EQUAL(bucketsTotal, static_cast<size_t>(stats["total"].asLong())); + CPPUNIT_ASSERT_EQUAL(bucketsPending, static_cast<size_t>(stats["pending"].asLong())); +} + +void +DistributorHostInfoReporterTest::verifyBucketSpaceStats(const vespalib::Slime& root, + uint16_t nodeIndex, + const vespalib::string& bucketSpaceName) { + const auto &stats = getBucketSpaceStats(root, nodeIndex, bucketSpaceName); + CPPUNIT_ASSERT(!stats["total"].valid()); + CPPUNIT_ASSERT(!stats["pending"].valid()); +} + +struct Fixture { MockedLatencyStatisticsProvider latencyStatsProvider; MockedMinReplicaProvider minReplicaProvider; - DistributorHostInfoReporter reporter(latencyStatsProvider, - minReplicaProvider); + MockedBucketSpacesStatsProvider bucketSpacesStatsProvider; + DistributorHostInfoReporter reporter; + Fixture() + : latencyStatsProvider(), + minReplicaProvider(), + bucketSpacesStatsProvider(), + reporter(latencyStatsProvider, minReplicaProvider, bucketSpacesStatsProvider) + {} + ~Fixture() {} +}; +void +DistributorHostInfoReporterTest::hostInfoWithPutLatenciesOnly() +{ + Fixture f; NodeStatsSnapshot snapshot; snapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; snapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; - latencyStatsProvider.returnedSnapshot = snapshot; + f.latencyStatsProvider.returnedSnapshot = snapshot; vespalib::Slime root; - util::reporterToSlime(reporter, root); + util::reporterToSlime(f.reporter, root); verifyReportedNodeLatencies(root, 0, 10000, 3); verifyReportedNodeLatencies(root, 5, 25000, 7); } @@ -135,23 +196,19 @@ DistributorHostInfoReporterTest::hostInfoWithPutLatenciesOnly() void DistributorHostInfoReporterTest::hostInfoAllInfo() { - MockedLatencyStatisticsProvider latencyStatsProvider; - MockedMinReplicaProvider minReplicaProvider; - DistributorHostInfoReporter reporter(latencyStatsProvider, - minReplicaProvider); - + Fixture f; NodeStatsSnapshot latencySnapshot; latencySnapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; latencySnapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; - latencyStatsProvider.returnedSnapshot = latencySnapshot; + f.latencyStatsProvider.returnedSnapshot = latencySnapshot; std::unordered_map<uint16_t, uint32_t> minReplica; minReplica[0] = 2; minReplica[5] = 9; - minReplicaProvider.minReplica = minReplica; + f.minReplicaProvider.minReplica = minReplica; vespalib::Slime root; - util::reporterToSlime(reporter, root); + util::reporterToSlime(f.reporter, root); verifyReportedNodeLatencies(root, 0, 10000, 3); verifyReportedNodeLatencies(root, 5, 25000, 7); @@ -162,26 +219,28 @@ DistributorHostInfoReporterTest::hostInfoAllInfo() void DistributorHostInfoReporterTest::generateExampleJson() { - MockedLatencyStatisticsProvider latencyStatsProvider; - MockedMinReplicaProvider minReplicaProvider; - DistributorHostInfoReporter reporter(latencyStatsProvider, - minReplicaProvider); - + Fixture f; NodeStatsSnapshot snapshot; snapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; snapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; - latencyStatsProvider.returnedSnapshot = snapshot; + f.latencyStatsProvider.returnedSnapshot = snapshot; std::unordered_map<uint16_t, uint32_t> minReplica; minReplica[0] = 2; minReplica[5] = 9; - minReplicaProvider.minReplica = minReplica; + f.minReplicaProvider.minReplica = minReplica; + + PerNodeBucketSpacesStats stats; + stats[0]["default"] = BucketSpaceStats(11, 3); + stats[0]["global"] = BucketSpaceStats(13, 5); + stats[5]["default"] = BucketSpaceStats(17, 7); + f.bucketSpacesStatsProvider.stats = stats; vespalib::asciistream json; vespalib::JsonStream stream(json, true); stream << Object(); - reporter.report(stream); + f.reporter.report(stream); stream << End(); stream.finalize(); @@ -204,23 +263,46 @@ DistributorHostInfoReporterTest::generateExampleJson() void DistributorHostInfoReporterTest::noReportGeneratedIfDisabled() { - MockedLatencyStatisticsProvider latencyStatsProvider; - MockedMinReplicaProvider minReplicaProvider; - DistributorHostInfoReporter reporter(latencyStatsProvider, - minReplicaProvider); - reporter.enableReporting(false); + Fixture f; + f.reporter.enableReporting(false); NodeStatsSnapshot snapshot; snapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; snapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; - latencyStatsProvider.returnedSnapshot = snapshot; + f.latencyStatsProvider.returnedSnapshot = snapshot; vespalib::Slime root; - util::reporterToSlime(reporter, root); + util::reporterToSlime(f.reporter, root); CPPUNIT_ASSERT_EQUAL(size_t(0), root.get().children()); } -} // distributor -} // storage +void +DistributorHostInfoReporterTest::bucket_spaces_stats_are_reported() +{ + Fixture f; + PerNodeBucketSpacesStats stats; + stats[1]["default"] = BucketSpaceStats(11, 3); + stats[1]["global"] = BucketSpaceStats(13, 5); + stats[2]["default"] = BucketSpaceStats(17, 7); + stats[2]["global"] = BucketSpaceStats(); + stats[3]["default"] = BucketSpaceStats(19, 11); + f.bucketSpacesStatsProvider.stats = stats; + + vespalib::Slime root; + util::reporterToSlime(f.reporter, root); + verifyBucketSpaceStats(root, 1, "default", 11, 3); + verifyBucketSpaceStats(root, 1, "global", 13, 5); + verifyBucketSpaceStats(root, 2, "default", 17, 7); + verifyBucketSpaceStats(root, 2, "global"); + verifyBucketSpaceStats(root, 3, "default", 19, 11); + try { + verifyBucketSpaceStats(root, 3, "global"); + CPPUNIT_ASSERT(false); + } catch (const std::runtime_error &ex) { + CPPUNIT_ASSERT("No bucket space found with name global" == vespalib::string(ex.what())); + } +} + +} diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index 1640af0f871..d585c4d0d32 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -9,6 +9,7 @@ #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> #include <tests/distributor/distributortestutil.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/storage/config/config-stor-distributormanager.h> @@ -18,6 +19,7 @@ using document::test::makeDocumentBucket; using document::test::makeBucketSpace; +using document::FixedBucketSpaces; namespace storage { @@ -58,6 +60,9 @@ class Distributor_Test : public CppUnit::TestFixture, CPPUNIT_TEST(closing_aborts_priority_queued_client_requests); CPPUNIT_TEST_SUITE_END(); +public: + Distributor_Test(); + protected: void testOperationGeneration(); void testOperationsGeneratedAndStartedWithoutDuplicates(); @@ -89,9 +94,12 @@ protected: void internal_messages_are_started_in_fifo_order_batch(); void closing_aborts_priority_queued_client_requests(); + std::vector<document::BucketSpace> _bucketSpaces; + public: void setUp() override { createLinks(); + _bucketSpaces = getBucketSpaces(); }; void tearDown() override { @@ -197,6 +205,13 @@ private: CPPUNIT_TEST_SUITE_REGISTRATION(Distributor_Test); +Distributor_Test::Distributor_Test() + : CppUnit::TestFixture(), + DistributorTestUtil(), + _bucketSpaces() +{ +} + void Distributor_Test::testOperationGeneration() { @@ -752,19 +767,23 @@ void Distributor_Test::sendDownClusterStateCommand() { } void Distributor_Test::replyToSingleRequestBucketInfoCommandWith1Bucket() { - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); - CPPUNIT_ASSERT_EQUAL(api::MessageType::REQUESTBUCKETINFO, - _sender.commands[0]->getType()); - auto& bucketReq(static_cast<api::RequestBucketInfoCommand&>( - *_sender.commands[0])); - auto bucketReply = bucketReq.makeReply(); - // Make sure we have a bucket to route our remove op to, or we'd get - // an immediate reply anyway. - dynamic_cast<api::RequestBucketInfoReply&>(*bucketReply) - .getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry(document::BucketId(1, 1), - api::BucketInfo(20, 10, 12, 50, 60, true, true))); - _distributor->handleMessage(std::move(bucketReply)); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); + for (uint32_t i = 0; i < _sender.commands.size(); ++i) { + CPPUNIT_ASSERT_EQUAL(api::MessageType::REQUESTBUCKETINFO, + _sender.commands[i]->getType()); + auto& bucketReq(static_cast<api::RequestBucketInfoCommand&> + (*_sender.commands[i])); + auto bucketReply = bucketReq.makeReply(); + if (bucketReq.getBucketSpace() == FixedBucketSpaces::default_space()) { + // Make sure we have a bucket to route our remove op to, or we'd get + // an immediate reply anyway. + dynamic_cast<api::RequestBucketInfoReply&>(*bucketReply) + .getBucketInfo().push_back( + api::RequestBucketInfoReply::Entry(document::BucketId(1, 1), + api::BucketInfo(20, 10, 12, 50, 60, true, true))); + } + _distributor->handleMessage(std::move(bucketReply)); + } _sender.commands.clear(); } diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp index fedd836513b..8aa9ffadebe 100644 --- a/storage/src/tests/distributor/distributortestutil.cpp +++ b/storage/src/tests/distributor/distributortestutil.cpp @@ -376,4 +376,14 @@ DistributorTestUtil::getDistribution() const { return getBucketSpaceRepo().get(makeBucketSpace()).getDistribution(); } +std::vector<document::BucketSpace> +DistributorTestUtil::getBucketSpaces() const +{ + std::vector<document::BucketSpace> res; + for (const auto &repo : getBucketSpaceRepo()) { + res.push_back(repo.first); + } + return res; +} + } diff --git a/storage/src/tests/distributor/distributortestutil.h b/storage/src/tests/distributor/distributortestutil.h index 19da0483165..45aaf6b1dc7 100644 --- a/storage/src/tests/distributor/distributortestutil.h +++ b/storage/src/tests/distributor/distributortestutil.h @@ -166,6 +166,7 @@ public: BucketDatabase::Entry getBucket(const document::BucketId& bId) const; + std::vector<document::BucketSpace> getBucketSpaces() const; protected: vdstestlib::DirConfig _config; std::unique_ptr<TestDistributorApp> _node; diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp index 945ccfa1484..7103a89229d 100644 --- a/storage/src/tests/distributor/idealstatemanagertest.cpp +++ b/storage/src/tests/distributor/idealstatemanagertest.cpp @@ -9,11 +9,13 @@ #include <vespa/storageapi/message/visitor.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <tests/distributor/distributortestutil.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/document/test/make_bucket_space.h> using document::test::makeDocumentBucket; using document::test::makeBucketSpace; +using document::FixedBucketSpaces; namespace storage { namespace distributor { @@ -22,9 +24,14 @@ class IdealStateManagerTest : public CppUnit::TestFixture, public DistributorTestUtil { public: - IdealStateManagerTest() {} + IdealStateManagerTest() + : CppUnit::TestFixture(), + DistributorTestUtil(), + _bucketSpaces() + {} void setUp() override { createLinks(); + _bucketSpaces = getBucketSpaces(); }; void tearDown() override { @@ -54,6 +61,9 @@ public: CPPUNIT_TEST(testBlockIdealStateOpsOnFullRequestBucketInfo); CPPUNIT_TEST(testBlockCheckForAllOperationsToSpecificBucket); CPPUNIT_TEST_SUITE_END(); +private: + std::vector<document::BucketSpace> _bucketSpaces; + std::string makeBucketStatusString(const std::string &defaultSpaceBucketStatus); }; CPPUNIT_TEST_SUITE_REGISTRATION(IdealStateManagerTest); @@ -91,8 +101,7 @@ IdealStateManagerTest::testStatusPage() { std::ostringstream ost; getIdealStateManager().getBucketStatus(ost); - CPPUNIT_ASSERT_EQUAL(std::string("<h2>default - BucketSpace(0x0000000000000001)</h2>\n" - "BucketId(0x4000000000000002) : [node(idx=0,crc=0xff,docs=10/10,bytes=10/10,trusted=true,active=true,ready=false)]<br>\n" + CPPUNIT_ASSERT_EQUAL(makeBucketStatusString("BucketId(0x4000000000000002) : [node(idx=0,crc=0xff,docs=10/10,bytes=10/10,trusted=true,active=true,ready=false)]<br>\n" "<b>BucketId(0x4000000000000005):</b> <i> : split: [Splitting bucket because its maximum size (200 b, 100 docs, 100 meta, 200 b total) is " "higher than the configured limit of (100, 1000000)]</i> [node(idx=0,crc=0xff,docs=100/100,bytes=200/200,trusted=true," "active=true,ready=false)]<br>\n"), @@ -113,8 +122,7 @@ IdealStateManagerTest::testDisabledStateChecker() { std::ostringstream ost; getIdealStateManager().getBucketStatus(ost); - CPPUNIT_ASSERT_EQUAL(std::string( - "<h2>default - BucketSpace(0x0000000000000001)</h2>\n" + CPPUNIT_ASSERT_EQUAL(makeBucketStatusString( "BucketId(0x4000000000000002) : [node(idx=0,crc=0xff,docs=10/10,bytes=10/10,trusted=true,active=true,ready=false)]<br>\n" "<b>BucketId(0x4000000000000005):</b> <i> : split: [Splitting bucket because its maximum size (200 b, 100 docs, 100 meta, 200 b total) is " "higher than the configured limit of (100, 1000000)]</i> [node(idx=0,crc=0xff,docs=100/100,bytes=200/200,trusted=true," @@ -261,6 +269,19 @@ IdealStateManagerTest::testBlockCheckForAllOperationsToSpecificBucket() } } +std::string +IdealStateManagerTest::makeBucketStatusString(const std::string &defaultSpaceBucketStatus) +{ + std::ostringstream ost; + for (const auto &bucketSpace : _bucketSpaces) { + ost << "<h2>" << FixedBucketSpaces::to_string(bucketSpace) << " - " << bucketSpace << "</h2>\n"; + if (bucketSpace == FixedBucketSpaces::default_space()) { + ost << defaultSpaceBucketStatus; + } + } + return ost.str(); +} + } // distributor } // storage diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index 48c24c06ea1..394df6024fd 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -60,7 +60,7 @@ void SimpleMaintenanceScannerTest::setUp() { _priorityGenerator.reset(new MockMaintenancePriorityGenerator()); - _bucketSpaceRepo = std::make_unique<DistributorBucketSpaceRepo>(false); + _bucketSpaceRepo = std::make_unique<DistributorBucketSpaceRepo>(); _priorityDb.reset(new SimpleBucketPriorityDatabase()); _scanner.reset(new SimpleMaintenanceScanner(*_priorityDb, *_priorityGenerator, *_bucketSpaceRepo)); } diff --git a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp index ac1a046bd17..7895d9e4cd0 100644 --- a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp +++ b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp @@ -140,6 +140,8 @@ ModifiedBucketCheckerTest::testDoNotCheckModifiedBucketsIfAlreadyPending() expectCommandsAndSendReplies(0, 0); // After replies received, tick should send new requests again. replyToAll(messages, 0); + _handler->tick(); // global bucket space ==> nothing to do + expectCommandsAndSendReplies(0, 0); _handler->tick(); expectCommandsAndSendReplies(3, 3); } @@ -177,13 +179,20 @@ ModifiedBucketCheckerTest::testRecheckRequestsAreChunked() _handler->tick(); expectCommandsAndSendReplies(1, 4); + _handler->tick(); // global bucket space ==> nothing to do + expectCommandsAndSendReplies(0, 0); + // New round of fetching _handler->tick(); expectCommandsAndSendReplies(1, 10); + _handler->tick(); // global bucket space ==> nothing to do + expectCommandsAndSendReplies(0, 0); // And done! _handler->tick(); expectCommandsAndSendReplies(0, 0); + _handler->tick(); // global bucket space ==> nothing to do + expectCommandsAndSendReplies(0, 0); } void diff --git a/storage/src/vespa/storage/common/content_bucket_space_repo.cpp b/storage/src/vespa/storage/common/content_bucket_space_repo.cpp index 304c6eddddc..774ddb81578 100644 --- a/storage/src/vespa/storage/common/content_bucket_space_repo.cpp +++ b/storage/src/vespa/storage/common/content_bucket_space_repo.cpp @@ -11,6 +11,7 @@ ContentBucketSpaceRepo::ContentBucketSpaceRepo() : _map() { _map.emplace(document::FixedBucketSpaces::default_space(), std::make_unique<ContentBucketSpace>(document::FixedBucketSpaces::default_space())); + _map.emplace(document::FixedBucketSpaces::global_space(), std::make_unique<ContentBucketSpace>(document::FixedBucketSpaces::global_space())); } ContentBucketSpace & @@ -21,10 +22,6 @@ ContentBucketSpaceRepo::get(BucketSpace bucketSpace) const return *itr->second; } -void ContentBucketSpaceRepo::enableGlobalBucketSpace() { - _map.emplace(document::FixedBucketSpaces::global_space(), std::make_unique<ContentBucketSpace>(document::FixedBucketSpaces::global_space())); -} - ContentBucketSpaceRepo::BucketSpaces ContentBucketSpaceRepo::getBucketSpaces() const { diff --git a/storage/src/vespa/storage/common/content_bucket_space_repo.h b/storage/src/vespa/storage/common/content_bucket_space_repo.h index f0803d76282..0d4ddb86bcf 100644 --- a/storage/src/vespa/storage/common/content_bucket_space_repo.h +++ b/storage/src/vespa/storage/common/content_bucket_space_repo.h @@ -24,8 +24,6 @@ public: BucketSpaceMap::const_iterator begin() const { return _map.begin(); } BucketSpaceMap::const_iterator end() const { return _map.end(); } - void enableGlobalBucketSpace(); - BucketSpaces getBucketSpaces() const; size_t getBucketMemoryUsage() const; diff --git a/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.h b/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.h new file mode 100644 index 00000000000..aeba827a976 --- /dev/null +++ b/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.h @@ -0,0 +1,48 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <map> +#include <unordered_map> + +namespace storage::distributor { + +/** + * Statistics for a single bucket space on a content node. + */ +class BucketSpaceStats { +private: + bool _valid; + size_t _bucketsTotal; + size_t _bucketsPending; +public: + BucketSpaceStats(size_t bucketsTotal_, size_t bucketsPending_) + : _valid(true), + _bucketsTotal(bucketsTotal_), + _bucketsPending(bucketsPending_) + {} + BucketSpaceStats() + : _valid(false), + _bucketsTotal(0), + _bucketsPending(0) + {} + bool valid() const { return _valid; } + size_t bucketsTotal() const { return _bucketsTotal; } + size_t bucketsPending() const { return _bucketsPending; } +}; + +/** + * Interface that provides snapshots of bucket spaces statistics per content node. + */ +class BucketSpacesStatsProvider { +public: + // Mapping from bucket space name to statistics for that bucket space. + using BucketSpacesStats = std::map<vespalib::string, BucketSpaceStats>; + // Mapping from content node index to statistics for all bucket spaces on that node. + using PerNodeBucketSpacesStats = std::unordered_map<uint16_t, BucketSpacesStats>; + + virtual ~BucketSpacesStatsProvider() {} + virtual PerNodeBucketSpacesStats getBucketSpacesStats() const = 0; +}; + +} diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index a559ce2ad1a..32554d02397 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -65,7 +65,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg, framework::StatusReporter("distributor", "Distributor"), _compReg(compReg), _component(compReg, "distributor"), - _bucketSpaceRepo(std::make_unique<DistributorBucketSpaceRepo>(_component.enableMultipleBucketSpaces())), + _bucketSpaceRepo(std::make_unique<DistributorBucketSpaceRepo>()), _metrics(new DistributorMetricSet(_component.getLoadTypes()->getMetricLoadTypes())), _operationOwner(*this, _component.getClock()), _maintenanceOperationOwner(*this, _component.getClock()), @@ -94,7 +94,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg, _metricLock(), _maintenanceStats(), _bucketDbStats(), - _hostInfoReporter(_pendingMessageTracker.getLatencyStatisticsProvider(), *this), + _hostInfoReporter(_pendingMessageTracker.getLatencyStatisticsProvider(), *this, *this), _ownershipSafeTimeCalc( std::make_unique<OwnershipTransferSafeTimePointCalculator>( std::chrono::seconds(0))) // Set by config later @@ -527,10 +527,8 @@ Distributor::propagateDefaultDistribution( std::shared_ptr<const lib::Distribution> distribution) { _bucketSpaceRepo->get(document::FixedBucketSpaces::default_space()).setDistribution(distribution); - if (_component.enableMultipleBucketSpaces()) { - auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); - _bucketSpaceRepo->get(document::FixedBucketSpaces::global_space()).setDistribution(std::move(global_distr)); - } + auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); + _bucketSpaceRepo->get(document::FixedBucketSpaces::global_space()).setDistribution(std::move(global_distr)); } void diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index 39695b26415..1cf0a4f1866 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -2,27 +2,27 @@ #pragma once -#include "idealstatemanager.h" +#include "bucket_spaces_stats_provider.h" #include "bucketdbupdater.h" -#include "pendingmessagetracker.h" +#include "distributor_host_info_reporter.h" +#include "distributorinterface.h" #include "externaloperationhandler.h" +#include "idealstatemanager.h" #include "min_replica_provider.h" -#include "distributorinterface.h" - +#include "pendingmessagetracker.h" #include "statusreporterdelegate.h" -#include "distributor_host_info_reporter.h" -#include <vespa/storage/distributor/maintenance/maintenancescheduler.h> -#include <vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h> +#include <vespa/config/config.h> #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/storage/common/messagesender.h> +#include <vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h> +#include <vespa/storage/distributor/maintenance/maintenancescheduler.h> #include <vespa/storageapi/message/state.h> -#include <vespa/storageframework/generic/thread/tickingthread.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> -#include <vespa/config/config.h> +#include <vespa/storageframework/generic/thread/tickingthread.h> #include <vespa/vespalib/util/sync.h> -#include <unordered_map> #include <queue> +#include <unordered_map> namespace storage { @@ -43,7 +43,8 @@ class Distributor : public StorageLink, public StatusDelegator, public framework::StatusReporter, public framework::TickingThread, - public MinReplicaProvider + public MinReplicaProvider, + public BucketSpacesStatsProvider { public: Distributor(DistributorComponentRegister&, @@ -197,6 +198,11 @@ private: */ std::unordered_map<uint16_t, uint32_t> getMinReplica() const override; + PerNodeBucketSpacesStats getBucketSpacesStats() const override { + // TODO: implement + return BucketSpacesStatsProvider::PerNodeBucketSpacesStats(); + } + /** * Atomically publish internal metrics to external ideal state metrics. * Takes metric lock. diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp index 117776cd242..cc1b1eb9a17 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp @@ -14,13 +14,11 @@ using document::BucketSpace; namespace storage { namespace distributor { -DistributorBucketSpaceRepo::DistributorBucketSpaceRepo(bool enableGlobalBucketSpace) +DistributorBucketSpaceRepo::DistributorBucketSpaceRepo() : _map() { add(document::FixedBucketSpaces::default_space(), std::make_unique<DistributorBucketSpace>()); - if (enableGlobalBucketSpace) { - add(document::FixedBucketSpaces::global_space(), std::make_unique<DistributorBucketSpace>()); - } + add(document::FixedBucketSpaces::global_space(), std::make_unique<DistributorBucketSpace>()); } DistributorBucketSpaceRepo::~DistributorBucketSpaceRepo() = default; diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h index 7d7db240ad9..e30438771b2 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h @@ -21,7 +21,7 @@ private: BucketSpaceMap _map; public: - explicit DistributorBucketSpaceRepo(bool enableGlobalBucketSpace); + DistributorBucketSpaceRepo(); ~DistributorBucketSpaceRepo(); DistributorBucketSpaceRepo(const DistributorBucketSpaceRepo&&) = delete; diff --git a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp index 8ac471ee0b7..5929f02c04b 100644 --- a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp +++ b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "bucket_spaces_stats_provider.h" #include "distributor_host_info_reporter.h" #include "min_replica_provider.h" #include "pendingmessagetracker.h" @@ -12,15 +13,19 @@ using std::unordered_map; namespace storage { namespace distributor { +using BucketSpacesStats = BucketSpacesStatsProvider::BucketSpacesStats; +using PerNodeBucketSpacesStats = BucketSpacesStatsProvider::PerNodeBucketSpacesStats; using Object = vespalib::JsonStream::Object; using Array = vespalib::JsonStream::Array; using End = vespalib::JsonStream::End; DistributorHostInfoReporter::DistributorHostInfoReporter( LatencyStatisticsProvider& latencyProvider, - MinReplicaProvider& minReplicaProvider) + MinReplicaProvider& minReplicaProvider, + BucketSpacesStatsProvider& bucketSpacesStatsProvider) : _latencyProvider(latencyProvider), _minReplicaProvider(minReplicaProvider), + _bucketSpacesStatsProvider(bucketSpacesStatsProvider), _enabled(true) { } @@ -38,15 +43,33 @@ writeOperationStats(vespalib::JsonStream& stream, } void +writeBucketSpacesStats(vespalib::JsonStream& stream, + const BucketSpacesStats& stats) +{ + for (const auto& elem : stats) { + stream << Object() << "name" << elem.first; + if (elem.second.valid()) { + stream << "total" << elem.second.bucketsTotal() + << "pending" << elem.second.bucketsPending(); + } + stream << End(); + } +} + +void outputStorageNodes(vespalib::JsonStream& output, const unordered_map<uint16_t, NodeStats>& nodeStats, - const unordered_map<uint16_t, uint32_t>& minReplica) + const unordered_map<uint16_t, uint32_t>& minReplica, + const PerNodeBucketSpacesStats& bucketSpacesStats) { set<uint16_t> nodes; - for (auto& element : nodeStats) { + for (const auto& element : nodeStats) { nodes.insert(element.first); } - for (auto& element : minReplica) { + for (const auto& element : minReplica) { + nodes.insert(element.first); + } + for (const auto& element : bucketSpacesStats) { nodes.insert(element.first); } @@ -69,6 +92,13 @@ outputStorageNodes(vespalib::JsonStream& output, output << "min-current-replication-factor" << minReplicaIt->second; } + + auto bucketSpacesStatsIt = bucketSpacesStats.find(node); + if (bucketSpacesStatsIt != bucketSpacesStats.end()) { + output << "bucket-spaces" << Array(); + writeBucketSpacesStats(output, bucketSpacesStatsIt->second); + output << End(); + } } output << End(); } @@ -83,15 +113,15 @@ DistributorHostInfoReporter::report(vespalib::JsonStream& output) return; } - NodeStatsSnapshot nodeStats = _latencyProvider.getLatencyStatistics(); - std::unordered_map<uint16_t, uint32_t> minReplica = - _minReplicaProvider.getMinReplica(); + auto nodeStats = _latencyProvider.getLatencyStatistics(); + auto minReplica = _minReplicaProvider.getMinReplica(); + auto bucketSpacesStats = _bucketSpacesStatsProvider.getBucketSpacesStats(); output << "distributor" << Object(); { output << "storage-nodes" << Array(); - outputStorageNodes(output, nodeStats.nodeToStats, minReplica); + outputStorageNodes(output, nodeStats.nodeToStats, minReplica, bucketSpacesStats); output << End(); } diff --git a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.h b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.h index 3cb878fc75c..3e6a02120c2 100644 --- a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.h +++ b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.h @@ -7,6 +7,7 @@ namespace storage { namespace distributor { +class BucketSpacesStatsProvider; class LatencyStatisticsProvider; class MinReplicaProvider; struct OperationStats; @@ -15,7 +16,8 @@ class DistributorHostInfoReporter : public HostReporter { public: DistributorHostInfoReporter(LatencyStatisticsProvider& latencyProvider, - MinReplicaProvider& minReplicaProvider); + MinReplicaProvider& minReplicaProvider, + BucketSpacesStatsProvider& bucketSpacesStatsProvider); DistributorHostInfoReporter(const DistributorHostInfoReporter&) = delete; DistributorHostInfoReporter& operator=( @@ -43,6 +45,7 @@ public: private: LatencyStatisticsProvider& _latencyProvider; MinReplicaProvider& _minReplicaProvider; + BucketSpacesStatsProvider& _bucketSpacesStatsProvider; std::atomic<bool> _enabled; }; diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp index 29c69b59760..888f1e816a1 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp @@ -42,18 +42,9 @@ void ServiceLayerComponentRegisterImpl::setDistribution(lib::Distribution::SP distribution) { _bucketSpaceRepo.get(document::FixedBucketSpaces::default_space()).setDistribution(distribution); - if (enableMultipleBucketSpaces()) { - auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); - _bucketSpaceRepo.get(document::FixedBucketSpaces::global_space()).setDistribution(global_distr); - } + auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); + _bucketSpaceRepo.get(document::FixedBucketSpaces::global_space()).setDistribution(global_distr); StorageComponentRegisterImpl::setDistribution(distribution); } -void ServiceLayerComponentRegisterImpl::setEnableMultipleBucketSpaces(bool enabled) { - StorageComponentRegisterImpl::setEnableMultipleBucketSpaces(enabled); - if (enabled) { - _bucketSpaceRepo.enableGlobalBucketSpace(); - } -} - } // storage diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h index fc07f394fff..deb3b2c0767 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h @@ -38,7 +38,6 @@ public: void registerServiceLayerComponent(ServiceLayerManagedComponent&) override; void setDiskCount(uint16_t count); void setDistribution(lib::Distribution::SP distribution) override; - void setEnableMultipleBucketSpaces(bool enabled) override; }; } // storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp index 6d1dc8d587b..30d6f5bd45c 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp @@ -5,6 +5,7 @@ #include "storagereply.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/util/stringutil.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <sstream> #include <vespa/log/bufferedlogger.h> @@ -42,6 +43,14 @@ namespace { vespalib::Version version5_0beta(4, 3, 0); } + +static bool +suppressEncodeWarning(const api::StorageMessage *msg) +{ + const auto *req = dynamic_cast<const api::RequestBucketInfoCommand *>(msg); + return ((req != nullptr) && (req->getBucketSpace() != document::FixedBucketSpaces::default_space())); +} + static mbus::Blob encodeMessage(const ProtocolSerialization & serializer, const mbus::Routable & routable, @@ -110,10 +119,13 @@ StorageProtocol::encode(const vespalib::Version& version, } } catch (std::exception & e) { - LOGBP(warning, "Failed to encode %s storage protocol message %s: %s", - version.toString().c_str(), - message.getInternalMessage()->toString().c_str(), - e.what()); + if (!(version < version6_0 && + suppressEncodeWarning(message.getInternalMessage().get()))) { + LOGBP(warning, "Failed to encode %s storage protocol message %s: %s", + version.toString().c_str(), + message.getInternalMessage()->toString().c_str(), + e.what()); + } } return mbus::Blob(0); diff --git a/vespabase/conf/default-env.txt.in b/vespabase/conf/default-env.txt.in index 38a4d0cded5..5b144e4c301 100644 --- a/vespabase/conf/default-env.txt.in +++ b/vespabase/conf/default-env.txt.in @@ -1,4 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. fallback VESPA_HOME @CMAKE_INSTALL_PREFIX@ override VESPA_USER vespa -override cloudconfig_server__disable_filedistributor true |