diff options
51 files changed, 369 insertions, 257 deletions
diff --git a/application/pom.xml b/application/pom.xml index c5e8136628d..b03c1aa8eed 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -92,6 +92,10 @@ <groupId>org.antlr</groupId> <artifactId>antlr4-runtime</artifactId> </dependency> + <dependency> + <groupId>org.hdrhistogram</groupId> + <artifactId>HdrHistogram</artifactId> + </dependency> </dependencies> <build> diff --git a/build_settings.cmake b/build_settings.cmake index cb613688941..e6483785126 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -23,8 +23,14 @@ set(C_WARN_OPTS "-Winline -Wuninitialized -Werror -Wall -W -Wchar-subscripts -Wc # Note: this is not a union of C_WARN_OPTS, since CMAKE_CXX_FLAGS already includes CMAKE_C_FLAGS, which in turn includes C_WARN_OPTS transitively set(CXX_SPECIFIC_WARN_OPTS "-Wsuggest-override -Wnon-virtual-dtor") +# Select C++ ABI +if(DEFINED VESPA_CXX_ABI_FLAGS) +else() +set (VESPA_CXX_ABI_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0") +endif() + # C and C++ compiler flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 ${C_WARN_OPTS} -fPIC -D_GLIBCXX_USE_CXX11_ABI=0 -DBOOST_DISABLE_ASSERTS -march=westmere -mtune=intel ${EXTRA_C_FLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DBOOST_DISABLE_ASSERTS -march=westmere -mtune=intel ${EXTRA_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} ${CXX_SPECIFIC_WARN_OPTS} -std=c++1z -fvisibility-inlines-hidden -fdiagnostics-color=auto ${EXTRA_CXX_FLAGS}") # Linker flags diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index 7c32c286fba..99c079739b4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -4,11 +4,9 @@ package com.yahoo.config.model.provision; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.provision.*; import com.yahoo.net.HostName; -import com.yahoo.vespa.model.HostSystem; import java.util.ArrayList; import java.util.List; -import java.net.UnknownHostException; /** * A host provisioner used when there is no hosts.xml file (using localhost as the only host) @@ -25,13 +23,8 @@ public class SingleNodeProvisioner implements HostProvisioner { private int counter = 0; public SingleNodeProvisioner() { - try { - host = new Host(HostSystem.lookupCanonicalHostname(HostName.getLocalhost())); - this.hostSpec = new HostSpec(host.hostname(), host.aliases()); - } - catch (UnknownHostException e) { - throw new RuntimeException(e); - } + host = new Host(HostName.getLocalhost()); + this.hostSpec = new HostSpec(host.hostname(), host.aliases()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index 27d5991154c..e4e1e029a1a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -50,7 +50,7 @@ public final class Host extends AbstractConfigProducer<AbstractConfigProducer<?> deployLogger().log(Level.WARNING, "Unable to find canonical hostname of host: " + hostname); } if ((null != canonical) && (! hostname.equals(canonical))) { - deployLogger().log(Level.WARNING, "Host named '" + hostname + "' will not receive any config " + + deployLogger().log(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + "since it does not match its canonical hostname: " + canonical); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index cc3f4a22966..e597e338314 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -95,7 +95,7 @@ public class VespaModelTestCase { LogdConfig.Builder b = new LogdConfig.Builder(); b = (LogdConfig.Builder) model.getConfig(b, ""); LogdConfig c = new LogdConfig(b); - assertEquals(HostSystem.lookupCanonicalHostname(HostName.getLocalhost()), c.logserver().host()); + assertEquals(HostName.getLocalhost(), c.logserver().host()); SlobroksConfig.Builder sb = new SlobroksConfig.Builder(); sb = (com.yahoo.cloud.config.SlobroksConfig.Builder) model.getConfig(sb, ""); @@ -106,7 +106,7 @@ public class VespaModelTestCase { zb = (ZookeepersConfig.Builder) model.getConfig(zb, ""); ZookeepersConfig zc = new ZookeepersConfig(zb); assertEquals(zc.zookeeperserverlist().split(",").length, 2); - assertTrue(zc.zookeeperserverlist().startsWith(HostSystem.lookupCanonicalHostname(HostName.getLocalhost()))); + assertTrue(zc.zookeeperserverlist().startsWith(HostName.getLocalhost())); ApplicationIdConfig.Builder appIdBuilder = new ApplicationIdConfig.Builder(); appIdBuilder = (ApplicationIdConfig.Builder) model.getConfig(appIdBuilder, ""); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationListener.java new file mode 100644 index 00000000000..bea25183eef --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationListener.java @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.application.ApplicationSet; + +public interface ApplicationListener { + /** + * Configs has been activated for an application: Either an application + * has been deployed for the first time, or it has been externally or internally redeployed. + * + * Must be thread-safe. + */ + void configActivated(TenantName tenant, ApplicationSet application); + + /** + * Application has been removed. + * + * Must be thread-safe. + */ + void applicationRemoved(ApplicationId applicationId); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java index 228ae0adfae..e392472480b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java @@ -15,16 +15,7 @@ import java.util.Collection; * @author lulf * @since 5.1 */ -public interface ReloadListener { - - /** - * Signal the listener that config has been reloaded. - * - * @param tenant Name of tenant for which config was reloaded. - * @param application the {@link com.yahoo.vespa.config.server.application.Application} that will be reloaded - */ - public void configReloaded(TenantName tenant, ApplicationSet application); - +public interface ReloadListener extends ApplicationListener { /** * Signal the listener that hosts used by by a particular tenant. * @@ -35,18 +26,10 @@ public interface ReloadListener { /** * Verify that given hosts are available for use by tenant. - * TODO: Does not belong here... * * @param tenant tenant that wants to allocate hosts. * @param newHosts a {@link java.util.Collection} of hosts that tenant wants to allocate. * @throws java.lang.IllegalArgumentException if one or more of the hosts are in use by another tenant. */ void verifyHostsAreAvailable(TenantName tenant, Collection<String> newHosts); - - /** - * Notifies listener that application with id {@link ApplicationId} has been removed. - * - * @param applicationId The {@link ApplicationId} of the removed application. - */ - void applicationRemoved(ApplicationId applicationId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java index 23c9bab450b..1532c05a56b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java @@ -68,7 +68,7 @@ public class SuperModelController { } } - SuperModel getSuperModel() { + public SuperModel getSuperModel() { return model; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelListener.java new file mode 100644 index 00000000000..81d706cf2e9 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelListener.java @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.model.SuperModel; + +/** + * Interface for those wanting to be notified about changes to the SuperModel. + */ +public interface SuperModelListener { + /** + * Application has been activated: Either deployed the first time, + * internally redeployed, or externally triggered redeploy. + */ + void applicationActivated(SuperModel superModel, Application application); + + /** + * Application has been removed. + */ + void applicationRemoved(SuperModel superModel, ApplicationId applicationId); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java new file mode 100644 index 00000000000..c3fd417f434 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -0,0 +1,108 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.config.server; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.GenerationCounter; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.model.SuperModel; + +import java.time.Instant; +import java.util.*; + +/** + * Provides a SuperModel - a model of all application instances, and makes it stays + * up to date as applications are added, redeployed, and removed. + */ +public class SuperModelManager implements SuperModelProvider, ApplicationListener { + private final Zone zone; + private SuperModel superModel; // Guarded by 'this' monitor + private final List<SuperModelListener> listeners = new ArrayList<>(); // Guarded by 'this' monitor + + // Generation of the super model + private long generation; + private final long masterGeneration; // ConfigserverConfig's generation + private final GenerationCounter generationCounter; + + @Inject + public SuperModelManager(ConfigserverConfig configserverConfig, + NodeFlavors nodeFlavors, + GenerationCounter generationCounter) { + this.zone = new Zone(configserverConfig, nodeFlavors); + this.generationCounter = generationCounter; + this.masterGeneration = configserverConfig.masterGeneration(); + makeNewSuperModel(new HashMap<>()); + } + + @Override + public synchronized SuperModel getSuperModel() { + return superModel; + } + + @Override + public synchronized long getGeneration() { + return generation; + } + + @Override + public synchronized void registerListener(SuperModelListener listener) { + listeners.add(listener); + } + + @Override + public synchronized void configActivated(TenantName tenant, ApplicationSet applicationSet) { + Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); + if (!newModels.containsKey(tenant)) { + // New application has been activated + newModels.put(tenant, new LinkedHashMap<>()); + } else { + // Application has been redeployed + } + + // TODO: Should supermodel care about multiple versions? + Application application = applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()); + newModels.get(tenant).put(applicationSet.getId(), application); + + makeNewSuperModel(newModels); + listeners.stream().forEach(listener -> listener.applicationActivated(superModel, application)); + } + + @Override + public synchronized void applicationRemoved(ApplicationId applicationId) { + Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); + if (newModels.containsKey(applicationId.tenant())) { + newModels.get(applicationId.tenant()).remove(applicationId); + if (newModels.get(applicationId.tenant()).isEmpty()) { + newModels.remove(applicationId.tenant()); + } + } + + makeNewSuperModel(newModels); + listeners.stream().forEach(listener -> listener.applicationRemoved(superModel, applicationId)); + } + + private void makeNewSuperModel(Map<TenantName, Map<ApplicationId, Application>> newModels) { + generation = masterGeneration + generationCounter.get(); + superModel = new SuperModel(newModels, zone); + } + + private Map<TenantName, Map<ApplicationId, Application>> createModelCopy() { + Map<TenantName, Map<ApplicationId, Application>> currentModels = superModel.applicationModels(); + Map<TenantName, Map<ApplicationId, Application>> newModels = new LinkedHashMap<>(); + for (Map.Entry<TenantName, Map<ApplicationId, Application>> entry : currentModels.entrySet()) { + Map<ApplicationId, Application> appMap = new LinkedHashMap<>(); + newModels.put(entry.getKey(), appMap); + for (Map.Entry<ApplicationId, Application> appEntry : entry.getValue().entrySet()) { + appMap.put(appEntry.getKey(), appEntry.getValue()); + } + } + + return newModels; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelProvider.java new file mode 100644 index 00000000000..81eb823c026 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelProvider.java @@ -0,0 +1,12 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.config.server; + +import com.yahoo.vespa.config.server.model.SuperModel; + +interface SuperModelProvider { + SuperModel getSuperModel(); + long getGeneration(); + + void registerListener(SuperModelListener listener); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index 345d952882d..95f16a7c1e7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -7,30 +7,26 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.Version; -import com.yahoo.config.provision.Zone; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.GenerationCounter; import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.GenerationCounter; import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.model.SuperModel; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactoryFactory; import java.io.IOException; -import java.time.Instant; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; import java.util.Set; /** - * Handles request for supermodel config + * Handles request for supermodel config. * * @author lulf * @since 5.9 @@ -39,82 +35,47 @@ public class SuperModelRequestHandler implements RequestHandler { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(SuperModelRequestHandler.class.getName()); private volatile SuperModelController handler; - private final GenerationCounter generationCounter; - private final Zone zone; - private final long masterGeneration; private final ConfigDefinitionRepo configDefinitionRepo; private final ConfigResponseFactory responseFactory; + private final SuperModelManager superModelManager; private volatile boolean enabled = false; /** * Creates a supermodel controller */ @Inject - public SuperModelRequestHandler(GenerationCounter generationCounter, ConfigDefinitionRepo configDefinitionRepo, - ConfigserverConfig configserverConfig, NodeFlavors nodeFlavors) { - this.generationCounter = generationCounter; + public SuperModelRequestHandler(ConfigDefinitionRepo configDefinitionRepo, + ConfigserverConfig configserverConfig, + SuperModelManager superModelManager) { this.configDefinitionRepo = configDefinitionRepo; - this.masterGeneration = configserverConfig.masterGeneration(); this.responseFactory = ConfigResponseFactoryFactory.createFactory(configserverConfig); - this.zone = new Zone(configserverConfig, nodeFlavors); - this.handler = createNewHandler(Collections.emptyMap()); + this.superModelManager = superModelManager; + updateHandler(); } /** * Signals that config has been reloaded for an {@link com.yahoo.vespa.config.server.application.Application} * belonging to a tenant. * - * TODO: This is a bit too complex I think. - * * @param tenant Name of tenant owning the application. * @param applicationSet The reloaded set of {@link com.yahoo.vespa.config.server.application.Application}. */ public synchronized void reloadConfig(TenantName tenant, ApplicationSet applicationSet) { - Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); - if (!newModels.containsKey(tenant)) { - newModels.put(tenant, new LinkedHashMap<>()); - } - // TODO: Should supermodel care about multiple versions? - newModels.get(tenant).put(applicationSet.getId(), applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now())); - handler = createNewHandler(newModels); + superModelManager.configActivated(tenant, applicationSet); + updateHandler(); } public synchronized void removeApplication(ApplicationId applicationId) { - Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); - if (newModels.containsKey(applicationId.tenant())) { - newModels.get(applicationId.tenant()).remove(applicationId); - if (newModels.get(applicationId.tenant()).isEmpty()) { - newModels.remove(applicationId.tenant()); - } - } - handler = createNewHandler(newModels); - } - - private SuperModelController createNewHandler(Map<TenantName, Map<ApplicationId, Application>> newModels) { - long generation = generationCounter.get() + masterGeneration; - SuperModel model = new SuperModel(newModels, zone); - return new SuperModelController(model, configDefinitionRepo, generation, responseFactory); - } - - private Map<TenantName, Map<ApplicationId, Application>> getCurrentModels() { - if (handler != null) { - return handler.getSuperModel().applicationModels(); - } else { - return new LinkedHashMap<>(); - } + superModelManager.applicationRemoved(applicationId); + updateHandler(); } - private Map<TenantName, Map<ApplicationId, Application>> createModelCopy() { - Map<TenantName, Map<ApplicationId, Application>> currentModels = getCurrentModels(); - Map<TenantName, Map<ApplicationId, Application>> newModels = new LinkedHashMap<>(); - for (Map.Entry<TenantName, Map<ApplicationId, Application>> entry : currentModels.entrySet()) { - Map<ApplicationId, Application> appMap = new LinkedHashMap<>(); - newModels.put(entry.getKey(), appMap); - for (Map.Entry<ApplicationId, Application> appEntry : entry.getValue().entrySet()) { - appMap.put(appEntry.getKey(), appEntry.getValue()); - } - } - return newModels; + private void updateHandler() { + handler = new SuperModelController( + superModelManager.getSuperModel(), + configDefinitionRepo, + superModelManager.getGeneration(), + responseFactory); } public SuperModelController getHandler() { return handler; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java index 1d7bcd8d001..eb41373aab8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.server.application.Application; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.Map; /** @@ -26,7 +27,7 @@ public class SuperModel implements LbServicesConfig.Producer, RoutingConfig.Prod private final Map<TenantName, Map<ApplicationId, Application>> models; private final LbServicesProducer lbProd; private final RoutingProducer zoneProd; - + public SuperModel(Map<TenantName, Map<ApplicationId, Application>> models, Zone zone) { this.models = models; this.lbProd = new LbServicesProducer(Collections.unmodifiableMap(models), zone); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 1a2433ec77e..4fe78a30344 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -187,7 +187,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { * This method should be called when config is reloaded in the server. */ @Override - public void configReloaded(TenantName tenant, ApplicationSet applicationSet) { + public void configActivated(TenantName tenant, ApplicationSet applicationSet) { ApplicationId applicationId = applicationSet.getId(); configReloaded(delayedConfigResponses.drainQueue(applicationId), Tenants.logPre(applicationId)); reloadSuperModel(tenant, applicationSet); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java index 20d1e88440f..93c26d532bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.tenant; import java.time.Clock; -import java.time.Instant; import java.util.*; import com.yahoo.config.provision.Version; @@ -81,7 +80,7 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host private void notifyReloadListeners(ApplicationSet applicationSet) { for (ReloadListener reloadListener : reloadListeners) { reloadListener.hostsUpdated(tenant, hostRegistry.getAllHosts()); - reloadListener.configReloaded(tenant, applicationSet); + reloadListener.configActivated(tenant, applicationSet); } } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 730f9cb3136..9330dae2e41 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -15,6 +15,7 @@ <component id="com.yahoo.vespa.config.server.session.FileDistributionFactory" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.SuperModelGenerationCounter" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.SuperModelManager" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.session.SessionPreparer" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.SuperModelRequestHandler" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.StaticConfigDefinitionRepo" bundle="configserver" /> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index dd19cfc29bb..4c746eba64e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -34,6 +34,7 @@ import static org.junit.Assert.*; public class SuperModelRequestHandlerTest { private static final File testApp = new File("src/test/resources/deploy/app"); + private SuperModelManager manager; private SuperModelGenerationCounter counter; private SuperModelRequestHandler controller; @@ -43,10 +44,9 @@ public class SuperModelRequestHandlerTest { @Before public void setup() throws IOException { counter = new SuperModelGenerationCounter(new MockCurator()); - controller = new SuperModelRequestHandler(counter, - new TestConfigDefinitionRepo(), - new ConfigserverConfig(new ConfigserverConfig.Builder()), - emptyNodeFlavors()); + ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); + manager = new SuperModelManager(configserverConfig, emptyNodeFlavors(), counter); + controller = new SuperModelRequestHandler(new TestConfigDefinitionRepo(), configserverConfig, manager); } @Test @@ -95,10 +95,9 @@ public class SuperModelRequestHandlerTest { public void test_super_model_master_generation() throws IOException, SAXException { TenantName tenantA = TenantName.from("a"); long masterGen = 10; - controller = new SuperModelRequestHandler(counter, - new TestConfigDefinitionRepo(), - new ConfigserverConfig(new ConfigserverConfig.Builder().masterGeneration(masterGen)), - emptyNodeFlavors()); + ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder().masterGeneration(masterGen)); + manager = new SuperModelManager(configserverConfig, emptyNodeFlavors(), counter); + controller = new SuperModelRequestHandler(new TestConfigDefinitionRepo(), configserverConfig, manager); long gen = counter.increment(); controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3L, 1)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index d66ddfade34..ef742ae3d38 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -75,7 +75,7 @@ public class RpcServerTest extends TestWithRpc { generationCounter.increment(); Application app = new Application(new VespaModel(MockApplicationPackage.createEmpty()), new ServerCache(), 2l, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), ApplicationId.defaultId()); ApplicationSet appSet = ApplicationSet.fromSingle(app); - rpcServer.configReloaded(TenantName.defaultName(), appSet); + rpcServer.configActivated(TenantName.defaultName(), appSet); ConfigKey<?> key = new ConfigKey<>(LbServicesConfig.class, "*"); JRTClientConfigRequest clientReq = JRTClientConfigRequestV3.createFromRaw(new RawConfig(key, LbServicesConfig.CONFIG_DEF_MD5), 120_000, Trace.createDummy(), CompressionType.UNCOMPRESSED, Optional.empty()); assertTrue(clientReq.validateParameters()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java index 52102e805b8..fa6adb64a8a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java @@ -11,12 +11,9 @@ import com.yahoo.jrt.Transport; import com.yahoo.net.HostName; import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.GenerationCounter; -import com.yahoo.vespa.config.server.SuperModelRequestHandler; +import com.yahoo.vespa.config.server.*; import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker; import com.yahoo.vespa.config.server.host.HostRegistries; -import com.yahoo.vespa.config.server.MemoryGenerationCounter; -import com.yahoo.vespa.config.server.PortRangeAllocator; -import com.yahoo.vespa.config.server.TestConfigDefinitionRepo; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.tenant.MockTenantProvider; import org.junit.After; @@ -82,11 +79,14 @@ public class TestWithRpc { } protected void createAndStartRpcServer(boolean hostedVespa) { + ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder().rpcport(port).numthreads(1).maxgetconfigclients(1).hostedVespa(hostedVespa)), - new SuperModelRequestHandler(generationCounter, - new TestConfigDefinitionRepo(), - new ConfigserverConfig(new ConfigserverConfig.Builder()), - emptyNodeFlavors()), + new SuperModelRequestHandler(new TestConfigDefinitionRepo(), + configserverConfig, + new SuperModelManager( + configserverConfig, + emptyNodeFlavors(), + generationCounter)), Metrics.createTestMetrics(), new HostRegistries(), hostLivenessTracker); rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java index 3b67597c43c..91cf6e79165 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java @@ -242,7 +242,7 @@ public class TenantRequestHandlerTest extends TestWithCurator { public AtomicInteger removed = new AtomicInteger(0); public Map<String, Collection<String>> tenantHosts = new LinkedHashMap<>(); @Override - public void configReloaded(TenantName tenant, ApplicationSet application) { + public void configActivated(TenantName tenant, ApplicationSet application) { reloaded.incrementAndGet(); } diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 7eef70f59e0..1fb1c9b1bd2 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -42,6 +42,12 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>simplemetrics</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>org.hdrhistogram</groupId> + <artifactId>HdrHistogram</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -97,6 +103,12 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>container-jersey2</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>org.ow2.asm</groupId> + <artifactId>asm</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/Athens.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/Athens.java index c1f72fa4370..0635fb850b7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/Athens.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/Athens.java @@ -17,7 +17,6 @@ public interface Athens { AthensPrincipal principalFrom(UserId userId); NTokenValidator validator(); NToken nTokenFrom(String rawToken); - UnauthorizedZmsClient unauthorizedZmsClient(); ZmsClientFactory zmsClientFactory(); AthensDomain screwdriverDomain(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/UnauthorizedZmsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/UnauthorizedZmsClient.java deleted file mode 100644 index d1996bdbd45..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/UnauthorizedZmsClient.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.athens; - -import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; - -import java.util.List; - -/** - * @author gv - */ -public class UnauthorizedZmsClient { - - private final ZmsClient client; - - public UnauthorizedZmsClient(ZmsClientFactory zmsClientFactory) { - client = zmsClientFactory.createClientWithoutPrincipal(); - } - - public List<AthensDomain> getDomainList(String prefix) { - return client.getDomainList(prefix); - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClientFactory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClientFactory.java index 24a2d67ebf6..e00f3ce2f64 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClientFactory.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClientFactory.java @@ -8,6 +8,4 @@ public interface ZmsClientFactory { ZmsClient createClientWithServicePrincipal(); ZmsClient createClientWithAuthorizedServiceToken(NToken authorizedServiceToken); - - ZmsClient createClientWithoutPrincipal(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensMock.java index a993c6e3da3..8bed95bfcd4 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensMock.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; import com.yahoo.vespa.hosted.controller.api.integration.athens.InvalidTokenException; import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; import com.yahoo.vespa.hosted.controller.api.integration.athens.NTokenValidator; -import com.yahoo.vespa.hosted.controller.api.integration.athens.UnauthorizedZmsClient; import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory; /** @@ -23,12 +22,10 @@ public class AthensMock extends AbstractComponent implements Athens { private static final AthensDomain screwdriverDomain = new AthensDomain("screwdriver-domain"); private final ZmsClientFactory zmsClientFactory; - private final UnauthorizedZmsClient unauthorizedZmsClient; private final NTokenValidator nTokenValidator; public AthensMock(AthensDbMock athensDb, NTokenValidator nTokenValidator) { this.zmsClientFactory = new ZmsClientFactoryMock(athensDb); - this.unauthorizedZmsClient = new UnauthorizedZmsClient(zmsClientFactory); this.nTokenValidator = nTokenValidator; } @@ -67,11 +64,6 @@ public class AthensMock extends AbstractComponent implements Athens { } @Override - public UnauthorizedZmsClient unauthorizedZmsClient() { - return unauthorizedZmsClient; - } - - @Override public ZmsClientFactory zmsClientFactory() { return zmsClientFactory; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientFactoryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientFactoryMock.java index 73d971a27fe..8bc8b29fb4c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientFactoryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientFactoryMock.java @@ -42,12 +42,6 @@ public class ZmsClientFactoryMock extends AbstractComponent implements ZmsClient return new ZmsClientMock(athens); } - @Override - public ZmsClient createClientWithoutPrincipal() { - log("createClientWithoutPrincipal()"); - return new ZmsClientMock(athens); - } - private static void log(String format, Object... args) { log.log(Level.INFO, String.format(format, args)); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingIssues.java index 3c9488e04d3..4801f551307 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingIssues.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingIssues.java @@ -89,7 +89,7 @@ public class LoggingIssues implements Issues { if (info != null) // we still remember this issue return info; else // we forgot this issue (due to restart) - recreate it here to avoid log noise - return file(issueId, new Issue("(Forgotten)", "")); + return file(issueId, new Issue("(Forgotten)", "(Forgotten)")); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 1b3821a12bf..1edaef6edb8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -22,6 +22,10 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; @@ -31,10 +35,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException; -import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; @@ -490,20 +490,30 @@ public class ApplicationController { } } + /** Deactivate application in the given zone */ + public Application deactivate(Application application, Zone zone) { + return deactivate(application, zone, Optional.empty(), false); + } + + /** Deactivate a known deployment of the given application */ public Application deactivate(Application application, Deployment deployment, boolean requireThatDeploymentHasExpired) { + return deactivate(application, deployment.zone(), Optional.of(deployment), requireThatDeploymentHasExpired); + } + + private Application deactivate(Application application, Zone zone, Optional<Deployment> deployment, + boolean requireThatDeploymentHasExpired) { try (Lock lock = lock(application.id())) { - // TODO: ignore no application errors for config server client, only return such errors from sherpa client. - if (requireThatDeploymentHasExpired && ! DeploymentExpirer.hasExpired(controller.zoneRegistry(), deployment, - clock.instant())) + if (deployment.isPresent() && requireThatDeploymentHasExpired && ! DeploymentExpirer.hasExpired( + controller.zoneRegistry(), deployment.get(), clock.instant())) { return application; + } try { - configserverClient.deactivate(new DeploymentId(application.id(), deployment.zone())); - } - catch (NoInstanceException e) { + configserverClient.deactivate(new DeploymentId(application.id(), zone)); + } catch (NoInstanceException ignored) { // ok; already gone } - application = application.withoutDeploymentIn(deployment.zone()); + application = application.withoutDeploymentIn(zone); store(application, lock); return application; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index dcb54f13e4b..68912ac55ef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost; @@ -85,6 +86,7 @@ public class Controller extends AbstractComponent { private final MetricsService metricsService; private final Chef chefClient; private final Athens athens; + private final ZmsClient zmsClient; /** * Creates a controller @@ -140,6 +142,7 @@ public class Controller extends AbstractComponent { this.chefClient = chefClient; this.clock = clock; this.athens = athens; + this.zmsClient = athens.zmsClientFactory().createClientWithServicePrincipal(); applicationController = new ApplicationController(this, db, curator, rotationRepository, athens.zmsClientFactory(), nameService, configServerClient, routingGenerator, clock); @@ -154,7 +157,7 @@ public class Controller extends AbstractComponent { public ApplicationController applications() { return applicationController; } public List<AthensDomain> getDomainList(String prefix) { - return athens.unauthorizedZmsClient().getDomainList(prefix); + return zmsClient.getDomainList(prefix); } public Athens athens() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d701f3d57a0..6d1edc22b93 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -796,13 +796,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region) { Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName)); - + Zone zone = new Zone(Environment.from(environment), RegionName.from(region)); Deployment deployment = application.deployments().get(zone); - if (deployment == null) - return ErrorResponse.notFoundError("Could not deactivate: " + application + " is not deployed in " + zone); - - controller.applications().deactivate(application, deployment, false); + if (deployment == null) { + // Attempt to deactivate application even if the deployment is not known by the controller + controller.applications().deactivate(application, zone); + } else { + controller.applications().deactivate(application, deployment, false); + } // TODO: Change to return JSON return new StringResponse("Deactivated " + path(TenantResource.API_PATH, tenantName, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 97e1bac35c8..7d1700270ea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -194,6 +194,13 @@ public class ApplicationApiTest extends ControllerContainerTest { "", Request.Method.DELETE), "Deactivated tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); + + // DELETE (deactivate) a deployment is idempotent + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", + "", + Request.Method.DELETE), + "Deactivated tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); + // DELETE an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.DELETE), ""); diff --git a/dist/release-vespa-rpm.sh b/dist/release-vespa-rpm.sh index d5f969b34dc..95c0b5ce902 100755 --- a/dist/release-vespa-rpm.sh +++ b/dist/release-vespa-rpm.sh @@ -18,7 +18,7 @@ git checkout master git pull --rebase # Delete existing branch if exists and create new one -git push --delete origin &> /dev/null || true +git push --delete origin $RPM_BRANCH &> /dev/null || true git branch -D $RPM_BRANCH &> /dev/null || true git checkout -b $RPM_BRANCH $GITREF diff --git a/dist/vespa.spec b/dist/vespa.spec index c47373cccce..eae6a0b3817 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -15,29 +15,52 @@ License: Commercial URL: http://vespa.ai Source0: vespa-%{version}.tar.gz +%if 0%{?centos} BuildRequires: epel-release BuildRequires: centos-release-scl BuildRequires: devtoolset-6-gcc-c++ BuildRequires: devtoolset-6-libatomic-devel BuildRequires: devtoolset-6-binutils +%define _devtoolset_enable /opt/rh/devtoolset-6/enable +%endif +%if 0%{?fedora} +BuildRequires: gcc-c++ +BuildRequires: libatomic +%endif BuildRequires: Judy-devel +%if 0%{?centos} BuildRequires: cmake3 +BuildRequires: llvm3.9-devel +BuildRequires: vespa-boost-devel >= 1.59.0-6 +BuildRequires: vespa-zookeeper-c-client-devel >= 3.4.9-6 +%endif +%if 0%{?fedora} +BuildRequires: cmake >= 3.9.1 +%if 0%{?fc25} +BuildRequires: llvm-devel >= 3.9.1 +BuildRequires: boost-devel >= 1.60 +%endif +%if 0%{?fc26} +BuildRequires: llvm-devel >= 4.0 +BuildRequires: boost-devel >= 1.63 +%endif +BuildRequires: zookeeper-devel >= 3.4.9 +%endif BuildRequires: lz4-devel BuildRequires: libzstd-devel BuildRequires: zlib-devel BuildRequires: maven BuildRequires: libicu-devel -BuildRequires: llvm3.9-devel BuildRequires: java-1.8.0-openjdk-devel BuildRequires: openssl-devel BuildRequires: rpm-build BuildRequires: make -BuildRequires: vespa-boost-devel >= 1.59.0-6 BuildRequires: vespa-cppunit-devel >= 1.12.1-6 BuildRequires: vespa-libtorrent-devel >= 1.0.11-6 -BuildRequires: vespa-zookeeper-c-client-devel >= 3.4.9-6 BuildRequires: systemd -Requires: epel-release +%if 0%{?centos} +Requires: epel-release +%endif Requires: which Requires: initscripts Requires: valgrind @@ -46,13 +69,33 @@ Requires: lz4 Requires: libzstd Requires: zlib Requires: libicu +%if 0%{?centos} Requires: llvm3.9 +Requires: vespa-boost >= 1.59.0-6 +Requires: vespa-zookeeper-c-client >= 3.4.9-6 +%define _extra_link_directory /usr/lib64/llvm3.9/lib;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib +%define _extra_include_directory /usr/include/llvm3.9;/opt/vespa-boost/include;/opt/vespa-libtorrent/include;/opt/vespa-zookeeper-c-client/include;/opt/vespa-cppunit/include +%endif +%if 0%{?fedora} +%if 0%{?fc25} +Requires: llvm >= 3.9.1 +Requires: boost >= 1.60 +%endif +%if 0%{?fc26} +Requires: llvm >= 4.0 +Requires: boost >= 1.63 +%define _vespa_llvm_version 4.0 +%endif +Requires: zookeeper >= 3.4.9 +%define _extra_link_directory /opt/vespa-libtorrent/lib;/opt/vespa-cppunit/lib +%define _extra_include_directory /opt/vespa-libtorrent/include;/opt/vespa-cppunit/include +%define _vespa_boost_lib_suffix %{nil} +%define _vespa_cxx_abi_flags -D_GLIBCXX_USE_CXX11_ABI=1 +%endif Requires: java-1.8.0-openjdk Requires: openssl -Requires: vespa-boost >= 1.59.0-6 Requires: vespa-cppunit >= 1.12.1-6 Requires: vespa-libtorrent >= 1.0.11-6 -Requires: vespa-zookeeper-c-client >= 3.4.9-6 Requires(pre): shadow-utils # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private @@ -67,15 +110,19 @@ Vespa - The open big data serving engine %setup -q %build -source /opt/rh/devtoolset-6/enable || true +%if 0%{?_devtoolset_enable:1} +source %{_devtoolset_enable} || true +%endif sh bootstrap.sh java mvn -nsu -T 2C install -DskipTests -Dmaven.javadoc.skip=true cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ - -DEXTRA_LINK_DIRECTORY="/usr/lib64/llvm3.9/lib;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib" \ - -DEXTRA_INCLUDE_DIRECTORY="/usr/include/llvm3.9;/opt/vespa-boost/include;/opt/vespa-libtorrent/include;/opt/vespa-zookeeper-c-client/include;/opt/vespa-cppunit/include" \ - -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server" \ - -DCMAKE_BUILD_RPATH=%{_prefix}/lib64 \ + -DEXTRA_LINK_DIRECTORY="%{_extra_link_directory}" \ + -DEXTRA_INCLUDE_DIRECTORY="%{_extra_include_directory}" \ + -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}};/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server" \ + %{?_vespa_llvm_version:-DVESPA_LLVM_VERSION="%{_vespa_llvm_version}"} \ + %{?_vespa_boost_lib_suffix:-DVESPA_BOOST_LIB_SUFFIX="%{_vespa_boost_lib_suffix}"} \ + %{?_vespa_cxx_abi_flags:-DVESPA_CXX_ABI_FLAGS="%{_vespa_cxx_abi_flags}"} \ . make %{_smp_mflags} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java index 110cd1594de..9f3da91e4d3 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.net.HostName; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; /** * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> @@ -13,7 +13,7 @@ public class HostNameExpression extends Expression { @Override protected void doExecute(ExecutionContext ctx) { - ctx.setValue(new StringFieldValue(normalizeHostName(HostName.getLocalhost()))); + ctx.setValue(new StringFieldValue(normalizeHostName(getDefaults().vespaHostname()))); } @Override diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java index e30ac7fa340..27d7b49bb9c 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java @@ -7,8 +7,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; import org.junit.Test; -import java.net.InetAddress; -import java.net.UnknownHostException; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; import static org.junit.Assert.*; @@ -35,13 +34,13 @@ public class HostNameTestCase { } @Test - public void requireThatHostnameIsSet() throws UnknownHostException { + public void requireThatHostnameIsSet() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); new HostNameExpression().execute(ctx); FieldValue val = ctx.getValue(); assertTrue(val instanceof StringFieldValue); - assertEquals(HostNameExpression.normalizeHostName(InetAddress.getLocalHost().getHostName()), + assertEquals(HostNameExpression.normalizeHostName(getDefaults().vespaHostname()), ((StringFieldValue)val).getString()); } } diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json index 0faf1873446..1bfeab96b08 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json @@ -38,7 +38,6 @@ "convergedStateVersion": "6.138.18", "failCount": 44, "hardwareFailure": true, - "hardwareFailureType": "disk_smart", "hardwareFailureDescription": "disk_smart", "wantToRetire": false, "wantToDeprovision": false, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java index 2beabcd8e8f..d146e976121 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -30,7 +30,6 @@ import java.util.stream.Collectors; */ public class NodePatcher { - public static final String HARDWARE_FAILURE_TYPE = "hardwareFailureType"; public static final String HARDWARE_FAILURE_DESCRIPTION = "hardwareFailureDescription"; private final NodeFlavors nodeFlavors; private final Inspector inspector; @@ -79,14 +78,7 @@ public class NodePatcher { List<Node> children = nodeRepository.getChildNodes(node.hostname()); boolean modified = false; - // TODO: Remove when all clients have switched to hardwareFailureDescription - if (inspector.field(HARDWARE_FAILURE_TYPE).valid()) { - Optional<String> hardwareFailure = asOptionalString(inspector.field(HARDWARE_FAILURE_TYPE)); - modified = true; - children = children.stream() - .map(node -> node.with(node.status().withHardwareFailureDescription(hardwareFailure))) - .collect(Collectors.toList()); - } else if (inspector.field(HARDWARE_FAILURE_DESCRIPTION).valid()) { + if (inspector.field(HARDWARE_FAILURE_DESCRIPTION).valid()) { Optional<String> hardwareFailure = asOptionalString(inspector.field(HARDWARE_FAILURE_DESCRIPTION)); modified = true; children = children.stream() @@ -121,7 +113,6 @@ public class NodePatcher { case "flavor" : return node.with(nodeFlavors.getFlavorOrThrow(asString(value))); case HARDWARE_FAILURE_DESCRIPTION: - case HARDWARE_FAILURE_TYPE: return node.with(node.status().withHardwareFailureDescription(asOptionalString(value))); case "parentHostname" : return node.withParentHostname(asString(value)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index a7efc3ccd82..78bd2b5e165 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -168,8 +168,6 @@ class NodesResponse extends HttpResponse { }); object.setLong("failCount", node.status().failCount()); object.setBool("hardwareFailure", node.status().hardwareFailureDescription().isPresent()); - // TODO: Remove when all clients have switched to hardwareFailureDescription - node.status().hardwareFailureDescription().ifPresent(failure -> object.setString("hardwareFailureType", failure)); node.status().hardwareFailureDescription().ifPresent(failure -> object.setString("hardwareFailureDescription", failure)); object.setBool("wantToRetire", node.status().wantToRetire()); object.setBool("wantToDeprovision", node.status().wantToDeprovision()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 38d3bf46028..5f81013d8e1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -460,24 +460,6 @@ public class RestApiTest { assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), Optional.of(true)); } - // TODO: Remove when all clients have switched to hardwareFailureDescription - @Test - public void test_hardware_patching_of_docker_host_using_old_field() throws Exception { - assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com"), Optional.of(false)); - assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), Optional.of(false)); - - assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", - Utf8.toBytes("{" + - "\"hardwareFailureType\": \"memory_mcelog\"" + - "}" - ), - Request.Method.PATCH), - "{\"message\":\"Updated dockerhost2.yahoo.com\"}"); - - assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com"), Optional.of(true)); - assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), Optional.of(true)); - } - @Test public void test_node_patch_to_remove_docker_ready_fields() throws Exception { assertResponse(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index 1fa8feb4586..6106e25e75e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -38,7 +38,6 @@ "convergedStateVersion": "6.43.0", "failCount": 0, "hardwareFailure": true, - "hardwareFailureType": "memory_mcelog", "hardwareFailureDescription": "memory_mcelog", "wantToRetire" : true, "wantToDeprovision" : true, @@ -738,6 +738,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.hdrhistogram</groupId> + <artifactId>HdrHistogram</artifactId> + <version>2.1.8</version> + </dependency> + <dependency> <groupId>org.json</groupId> <artifactId>json</artifactId> <version>20090211</version> diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 6a5932d2b3a..0bbbdabd0aa 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -35,6 +35,7 @@ LOG_SETUP("attribute_manager_test"); #include <vespa/searchlib/predicate/predicate_index.h> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/threadstackexecutor.h> @@ -58,6 +59,7 @@ using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; using search::attribute::ImportedAttributeVector; using search::attribute::ReferenceAttribute; +using search::attribute::test::MockGidToLidMapperFactory; using search::index::DummyFileHeaderContext; using search::predicate::PredicateIndex; using search::predicate::PredicateTreeAnnotations; @@ -132,6 +134,7 @@ struct ImportedAttributesRepoBuilder { ImportedAttributesRepoBuilder() : _repo(std::make_unique<ImportedAttributesRepo>()) {} void add(const vespalib::string &name) { auto refAttr = std::make_shared<ReferenceAttribute>(name + "_ref", AVConfig(BasicType::REFERENCE)); + refAttr->setGidToLidMapperFactory(std::make_shared<MockGidToLidMapperFactory>()); auto targetAttr = search::AttributeFactory::createAttribute(name + "_target", INT32_SINGLE); auto documentMetaStore = std::shared_ptr<search::IDocumentMetaStoreContext>(); auto importedAttr = std::make_shared<ImportedAttributeVector>(name, refAttr, targetAttr, documentMetaStore, false); diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp index f221b2140c1..aac74e1583d 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp @@ -8,6 +8,7 @@ LOG_SETUP("imported_attributes_context_test"); #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> +#include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> #include <future> using namespace proton; @@ -17,12 +18,15 @@ using search::attribute::Config; using search::attribute::IAttributeVector; using search::attribute::ImportedAttributeVector; using search::attribute::ReferenceAttribute; +using search::attribute::test::MockGidToLidMapperFactory; using generation_t = AttributeVector::generation_t; ReferenceAttribute::SP createReferenceAttribute(const vespalib::string &name) { - return std::make_shared<ReferenceAttribute>(name, Config(BasicType::REFERENCE)); + auto refAttr = std::make_shared<ReferenceAttribute>(name, Config(BasicType::REFERENCE)); + refAttr->setGidToLidMapperFactory(std::make_shared<MockGidToLidMapperFactory>()); + return refAttr; } AttributeVector::SP diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp index 5445ba9c585..cec51e18e3c 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp @@ -37,10 +37,6 @@ BucketId toBucketId(const GlobalId &gid) { return bucketId; } -void assertLid(const std::unique_ptr<search::IGidToLidMapper> &mapper, const vespalib::string &docId, uint32_t lid) { - EXPECT_EQUAL(lid, mapper->mapGidToLid(toGid(docId))); -} - using GidMap = std::map<GlobalId, uint32_t>; struct GidCollector : public search::IGidToLidMapperVisitor @@ -66,6 +62,13 @@ void assertGids(const GidMap &expGids, const GidMap &gids) EXPECT_EQUAL(expGids, gids); } +void assertLid(const std::unique_ptr<search::IGidToLidMapper> &mapper, const vespalib::string &docId, uint32_t lid) { + auto gids = collectGids(mapper); + auto itr = gids.find(toGid(docId)); + uint32_t foundLid = (itr != gids.end()) ? itr->second : 0u; + EXPECT_EQUAL(lid, foundLid); +} + } struct Fixture @@ -139,15 +142,6 @@ struct Fixture } }; -TEST_F("Test that we can use gid mapper to get lids", Fixture) -{ - auto factory = f.getGidToLidMapperFactory(); - auto mapper = factory->getMapper(); - TEST_DO(assertLid(mapper, doc1, 4)); - TEST_DO(assertLid(mapper, doc2, 7)); - TEST_DO(assertLid(mapper, doc3, 0)); -} - TEST_F("Test that mapper holds read guard", Fixture) { TEST_DO(f.assertGenerations(3, 3)); diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.cpp index db990dff3bc..20bc1d1a49d 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.cpp @@ -16,17 +16,6 @@ GidToLidMapper::~GidToLidMapper() { } -uint32_t -GidToLidMapper::mapGidToLid(const document::GlobalId &gid) const -{ - uint32_t lid = 0; - if (_dms.getLid(gid, lid)) { - return lid; - } else { - return 0u; - } -} - void GidToLidMapper::foreach(const search::IGidToLidMapperVisitor &visitor) const { diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.h index ed6bf91dbd3..47bcf4257f8 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.h +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.h @@ -21,7 +21,6 @@ public: GidToLidMapper(vespalib::GenerationHandler::Guard &&guard, const DocumentMetaStore &dms); virtual ~GidToLidMapper(); - virtual uint32_t mapGidToLid(const document::GlobalId &gid) const override; virtual void foreach(const search::IGidToLidMapperVisitor &visitor) const override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index 2860769235d..c954c34aee8 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "imported_attribute_vector_read_guard.h" +#include <vespa/searchlib/common/i_gid_to_lid_mapper.h> +#include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h> namespace search { namespace attribute { @@ -17,7 +19,8 @@ ImportedAttributeVectorReadGuard::ImportedAttributeVectorReadGuard( _referencedLids(), _reference_attribute_guard(_reference_attribute), _target_attribute_guard(stableEnumGuard ? std::shared_ptr<AttributeVector>() : _target_attribute), - _target_attribute_enum_guard(stableEnumGuard ? _target_attribute : std::shared_ptr<AttributeVector>()) + _target_attribute_enum_guard(stableEnumGuard ? _target_attribute : std::shared_ptr<AttributeVector>()), + _mapper(_reference_attribute->getGidToLidMapperFactory()->getMapper()) { _referencedLids = _reference_attribute->getReferencedLids(); } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index 1937953df66..2ca8680f4b8 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -5,6 +5,8 @@ #include "imported_attribute_vector.h" #include "attributeguard.h" +namespace search { class IGidToLidMapper; } + namespace search::attribute { class BitVectorSearchCache; @@ -22,6 +24,7 @@ class ImportedAttributeVectorReadGuard : public ImportedAttributeVector AttributeGuard _reference_attribute_guard; AttributeGuard _target_attribute_guard; AttributeEnumGuard _target_attribute_enum_guard; + std::unique_ptr<IGidToLidMapper> _mapper; uint32_t getReferencedLid(uint32_t lid) const { return _referencedLids[lid]; diff --git a/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper.h b/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper.h index 8a9e2dbd84f..01431d0c75d 100644 --- a/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper.h +++ b/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper.h @@ -29,7 +29,6 @@ class IGidToLidMapper { public: virtual ~IGidToLidMapper() { } - virtual uint32_t mapGidToLid(const document::GlobalId &gid) const = 0; virtual void foreach(const IGidToLidMapperVisitor &visitor) const = 0; }; diff --git a/searchlib/src/vespa/searchlib/test/mock_gid_to_lid_mapping.h b/searchlib/src/vespa/searchlib/test/mock_gid_to_lid_mapping.h index 86203c19970..df7fff18bca 100644 --- a/searchlib/src/vespa/searchlib/test/mock_gid_to_lid_mapping.h +++ b/searchlib/src/vespa/searchlib/test/mock_gid_to_lid_mapping.h @@ -21,15 +21,6 @@ struct MockGidToLidMapper : public search::IGidToLidMapper { { } - uint32_t mapGidToLid(const document::GlobalId &gid) const override { - auto itr = _map.find(gid); - if (itr != _map.end()) { - return itr->second; - } else { - return 0u; - } - } - void foreach(const search::IGidToLidMapperVisitor &visitor) const override { for (const auto &kv : _map) { if (kv.second != 0) { diff --git a/simplemetrics/pom.xml b/simplemetrics/pom.xml index 1e06726e18d..d027fda82ce 100644 --- a/simplemetrics/pom.xml +++ b/simplemetrics/pom.xml @@ -20,7 +20,6 @@ <dependency> <groupId>org.hdrhistogram</groupId> <artifactId>HdrHistogram</artifactId> - <version>2.1.8</version> <scope>compile</scope> </dependency> <dependency> |