diff options
62 files changed, 568 insertions, 199 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 23dcef6d96d..6c557179750 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,6 +45,7 @@ add_subdirectory(container-disc) add_subdirectory(container-jersey2) add_subdirectory(container-messagebus) add_subdirectory(container-search) +add_subdirectory(container-search-gui) add_subdirectory(container-search-and-docproc) add_subdirectory(clustercontroller-apps) add_subdirectory(clustercontroller-apputil) diff --git a/README.md b/README.md index d84ebbe4456..acbeb6ffcec 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ Travis-CI build status: [. +C++ and Java building is supported on CentOS 7. The Java source can also be built on any platform having Java 8 and Maven installed. +We recommend using the following environment: [Create C++ / Java dev environment on CentOS using VirtualBox and Vagrant](vagrant/README.md). You can also setup CentOS 7 natively and install the following build dependencies: sudo yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo diff --git a/application/pom.xml b/application/pom.xml index db298451a8b..0808f2be58e 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -46,6 +46,11 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-search-gui</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>zkfacade</artifactId> <version>${project.version}</version> </dependency> diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java index da643c687d9..81f19f00fbb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java @@ -36,7 +36,7 @@ public class DomComponentBuilder extends VespaDomBuilder.DomConfigProducerBuilde private Component buildComponent(Element spec) { BundleInstantiationSpecification bundleSpec = - BundleInstantiationSpecificationBuilder.build(spec, false).nestInNamespace(namespace); + BundleInstantiationSpecificationBuilder.build(spec).nestInNamespace(namespace); return new Component<Component<?, ?>, ComponentModel>(new ComponentModel(bundleSpec)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java index 2c6364c4c34..74cc2b8421c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java @@ -13,6 +13,6 @@ import org.w3c.dom.Element; public class DomFilterBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Component> { @Override protected Component doBuild(AbstractConfigProducer ancestor, Element element) { - return new HttpFilter(BundleInstantiationSpecificationBuilder.build(element, false)); + return new HttpFilter(BundleInstantiationSpecificationBuilder.build(element)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 23f65fd9fb1..28a4b9dfea5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -12,18 +12,8 @@ import org.w3c.dom.Element; /** * @author gjoranv - * @since 5.1.6 */ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> { - private final boolean legacyMode; - - public DomHandlerBuilder(boolean legacyMode) { - this.legacyMode = legacyMode; - } - - public DomHandlerBuilder() { - this(false); - } @Override protected Handler doBuild(AbstractConfigProducer ancestor, Element handlerElement) { @@ -41,7 +31,7 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< } protected Handler<? super Component<?, ?>> getHandler(Element handlerElement) { - BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement, legacyMode); + BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement); return new Handler<>( new ComponentModel(bundleSpec)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java index 3d779289340..d2d9e9d2ca6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java @@ -27,7 +27,7 @@ public class ServletBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Ser } private SimpleComponent createServletComponent(Element servletElement) { - ComponentModel componentModel = new ComponentModel(BundleInstantiationSpecificationBuilder.build(servletElement, false)); + ComponentModel componentModel = new ComponentModel(BundleInstantiationSpecificationBuilder.build(servletElement)); return new SimpleComponent(componentModel); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java index 700b042a745..68b37d938a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java @@ -15,7 +15,7 @@ public class ChainedComponentModelBuilder extends GenericChainedComponentModelBu public ChainedComponentModelBuilder(Element spec) { super(spec); - bundleInstantiationSpec = BundleInstantiationSpecificationBuilder.build(spec, false); + bundleInstantiationSpec = BundleInstantiationSpecificationBuilder.build(spec); } public ChainedComponentModel build() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java new file mode 100644 index 00000000000..025075be8fd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java @@ -0,0 +1,27 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.Handler; + + +/** + * @author Henrik Høiness + */ + +public class GUIHandler extends Handler<AbstractConfigProducer<?>> { + public static final String BUNDLE = "container-search-gui"; + public static final String CLASS = "com.yahoo.search.query.gui.GUIHandler"; + public static final String BINDING = "*/querybuilder/*"; + + public GUIHandler() { + super(new ComponentModel(bundleSpec(CLASS, BUNDLE))); + } + + public static BundleInstantiationSpecification bundleSpec(String className, String bundle) { + return BundleInstantiationSpecification.getFromStrings(className, className, bundle); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index 69ee9048b28..2828fcf09d0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -4,11 +4,10 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.component.ComponentSpecification; -import com.yahoo.processing.handler.ProcessingHandler; -import com.yahoo.search.handler.SearchHandler; import org.w3c.dom.Element; -import java.util.*; +import java.util.Arrays; +import java.util.List; /** * This object builds a bundle instantiation spec from an XML element. @@ -17,14 +16,13 @@ import java.util.*; */ public class BundleInstantiationSpecificationBuilder { - public static BundleInstantiationSpecification build(Element spec, boolean legacyMode) { + public static BundleInstantiationSpecification build(Element spec) { ComponentSpecification id = XmlHelper.getIdRef(spec); ComponentSpecification classId = getComponentSpecification(spec, "class"); ComponentSpecification bundle = getComponentSpecification(spec, "bundle"); BundleInstantiationSpecification instSpec = new BundleInstantiationSpecification(id, classId, bundle); - if ( ! legacyMode) // TODO: Remove? - validate(instSpec); + validate(instSpec); return bundle == null ? setBundleForKnownClass(instSpec) : instSpec; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index a007c4765c0..d81026c54d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -23,6 +23,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.config.MetricDefaultsConfig; +import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; @@ -47,6 +48,7 @@ import com.yahoo.vespa.model.container.IdentityProvider; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; +import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChains; @@ -55,6 +57,7 @@ import com.yahoo.vespa.model.container.http.xml.HttpBuilder; import com.yahoo.vespa.model.container.jersey.xml.RestApiBuilder; import com.yahoo.vespa.model.container.processing.ProcessingChains; import com.yahoo.vespa.model.container.search.ContainerSearch; +import com.yahoo.vespa.model.container.search.GUIHandler; import com.yahoo.vespa.model.container.search.PageTemplates; import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.container.search.SemanticRules; @@ -378,6 +381,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.setSearch(buildSearch(cluster, searchElement, queryProfiles, semanticRules)); addSearchHandler(cluster, searchElement); + addGUIHandler(cluster); validateAndAddConfiguredComponents(cluster, searchElement, "renderer", ContainerModelBuilder::validateRendererElement); } } @@ -670,6 +674,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addComponent(searchHandler); } + private void addGUIHandler(ContainerCluster cluster) { + Handler<?> guiHandler = new GUIHandler(); + guiHandler.addServerBindings("http://"+GUIHandler.BINDING, "https://"+GUIHandler.BINDING); + cluster.addComponent(guiHandler); + } + + private String[] serverBindings(Element searchElement, String... defaultBindings) { List<Element> bindings = XML.getChildren(searchElement, "binding"); if (bindings.isEmpty()) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java index c58f3308ced..a651bbb7772 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -53,7 +53,7 @@ public class BundleInstantiationSpecificationBuilderTest { InputStream xmlStream = IOUtils.toInputStream(xml); Element component = XmlHelper.getDocumentBuilder().parse(xmlStream).getDocumentElement(); - BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component, false); + BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component); assertThat(spec.bundle, is(ComponentSpecification.fromString(expectedBundle))); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index cc8733fc9e2..30f1df6a394 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -8,6 +8,8 @@ import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.search.GUIHandler; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; @@ -32,6 +34,31 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { return root.getConfig(ChainsConfig.class, "default/component/com.yahoo.search.handler.SearchHandler"); } + @Test + public void gui_search_handler_is_always_included_when_search_is_specified() throws Exception{ + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <search />", + nodesXml, + "</jdisc>"); + + createModel(root, clusterElem); + + String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); + assertThat(discBindingsConfig, containsString(GUIHandler.BINDING)); + + ContainerCluster cluster = (ContainerCluster)root.getChildren().get("default"); + + GUIHandler guiHandler = null; + for (Handler<?> handler : cluster.getHandlers()) { + if (handler instanceof GUIHandler) { + guiHandler = (GUIHandler) handler; + } + } + if (guiHandler == null) fail(); + } + + @Test public void search_handler_bindings_can_be_overridden() throws Exception { diff --git a/config/src/apps/vespa-get-config/getconfig.cpp b/config/src/apps/vespa-get-config/getconfig.cpp index 4c42779ce91..9e58c9e3777 100644 --- a/config/src/apps/vespa-get-config/getconfig.cpp +++ b/config/src/apps/vespa-get-config/getconfig.cpp @@ -49,7 +49,8 @@ GetConfig::usage() fprintf(stderr, "usage: %s -n name -i configId\n", _argv[0]); fprintf(stderr, "-n name (config name, including namespace, on the form <namespace>.<name>)\n"); fprintf(stderr, "-i configId (config id, optional)\n"); - fprintf(stderr, "-j (output config as json)\n"); + fprintf(stderr, "-j (output config as json, optional)\n"); + fprintf(stderr, "-l (output config in legacy cfg format, optional)\n"); fprintf(stderr, "-a schema (config def schema file, optional)\n"); fprintf(stderr, "-v defVersion (config definition version, optional, deprecated)\n"); fprintf(stderr, "-m defMd5 (definition md5sum, optional)\n"); @@ -135,6 +136,9 @@ GetConfig::Main() case 'j': printAsJson = true; break; + case 'l': + printAsJson = false; + break; case 'm': defMD5 = optArg; break; diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index daaf8906d24..70df8ce5164 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -55,7 +55,7 @@ ztsUrl string default="" nodeAdminInContainer bool default=true # Maintainers -maintainerIntervalMinutes int default=30 +maintainerIntervalMinutes int default=60 # TODO: Default set to a high value (1 year) => maintainer will not run, change when maintainer verified out in prod tenantsMaintainerIntervalMinutes int default=525600 diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 4380cf5f047..b14fe5bcac9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -97,7 +97,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final Clock clock; private final DeployLogger logger = new SilentDeployLogger(); private final ConfigserverConfig configserverConfig; - private final Environment environment; private final FileDistributionStatus fileDistributionStatus; @Inject @@ -139,7 +138,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.httpProxy = httpProxy; this.clock = clock; this.configserverConfig = configserverConfig; - this.environment = Environment.from(configserverConfig.environment()); this.fileDistributionStatus = fileDistributionStatus; } @@ -250,7 +248,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // Keep manually deployed tenant applications on the latest version, don't change version otherwise // TODO: Remove this and always use version from session once controller starts upgrading manual deployments - Version version = decideVersion(application, environment, newSession.getVespaVersion(), bootstrap); + Version version = decideVersion(application, zone().environment(), newSession.getVespaVersion(), bootstrap); return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock, false /* don't validate as this is already deployed */, version, @@ -292,15 +290,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * @throws RuntimeException if the delete transaction fails. This method is exception safe. */ public boolean delete(ApplicationId applicationId) { - Zone zone = new Zone(Environment.from(configserverConfig.environment()), RegionName.from(configserverConfig.region())); List<String> hostedZonesToUseDeleteApplication = Arrays.asList("dev.us-east-1", "dev.corp-us-east-1", "test.us-east-1", "prod.corp-us-east-1", "prod.aws-us-east-1a", "prod.aws-us-west-1b"); - boolean useDeleteApplicationLegacyInThisZone = !hostedZonesToUseDeleteApplication.contains(zone.toString()); + boolean useDeleteApplicationLegacyInThisZone = !hostedZonesToUseDeleteApplication.contains(zone().toString()); // TODO: Use deleteApplication() in all zones if (configserverConfig.deleteApplicationLegacy() || - (configserverConfig.hostedVespa() && SystemName.from(configserverConfig.system()) == SystemName.main && + (configserverConfig.hostedVespa() && zone().system() == SystemName.main && useDeleteApplicationLegacyInThisZone)) { return deleteApplicationLegacy(applicationId); } else { @@ -561,10 +558,20 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return session.getSessionId(); } - public void deleteOldSessions() { + public void deleteExpiredLocalSessions() { listApplications().forEach(app -> tenantRepository.getTenant(app.tenant()).getLocalSessionRepo().purgeOldSessions()); } + public int deleteExpiredRemoteSessions(Duration expiryTime) { + return listApplications() + .stream() + .map(app -> tenantRepository.getTenant(app.tenant()).getRemoteSessionRepo() + // TODO: Delete in all zones + .deleteExpiredSessions(expiryTime, zone().system() == SystemName.cd)) + .mapToInt(i -> i) + .sum(); + } + // ---------------- Tenant operations ---------------------------------------------------------------- public Set<TenantName> deleteUnusedTenants(Duration ttlForUnusedTenant, Instant now) { @@ -608,7 +615,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return getLocalSession(tenant, sessionId).getMetaData(); } - ConfigserverConfig configserverConfig() { + public ConfigserverConfig configserverConfig() { return configserverConfig; } @@ -768,4 +775,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return deployLog; } + public Zone zone() { + return new Zone(SystemName.from(configserverConfig.system()), + Environment.from(configserverConfig.environment()), + RegionName.from(configserverConfig.region())); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java index 8ad7a85619c..7c962e5b6be 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.util.Optional; /** - * HTTP handler for a v2 getConfig operation + * HTTP handler for a v1 getConfig operation * * @author Ulf Lilleengen */ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java index 53f2553161f..a0cbf4e4845 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -14,13 +14,22 @@ import java.time.Duration; * @author hmusum */ public class SessionsMaintainer extends Maintainer { + private final boolean hostedVespa; SessionsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { super(applicationRepository, curator, interval); + this.hostedVespa = applicationRepository.configserverConfig().hostedVespa(); } @Override protected void maintain() { - applicationRepository.deleteOldSessions(); + applicationRepository.deleteExpiredLocalSessions(); + + // Expired remote sessions are not expected to exist, they should have been deleted when + // a deployment happened or when the application was deleted. We still see them from time to time, + // probably due to some race or another bug + Duration expiryTime = Duration.ofDays(30); + if (hostedVespa) + applicationRepository.deleteExpiredRemoteSessions(expiryTime); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java index 625665312fd..d01181638c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java @@ -6,22 +6,27 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.curator.Curator; import java.time.Duration; +import java.util.Arrays; +import java.util.List; /** - * Removes unused zookeeper data (for now only data used by old file distribution code is removed) + * Removes unused zookeeper data * * @author hmusum */ public class ZooKeeperDataMaintainer extends Maintainer { + private static final List<String> pathsToDelete = Arrays.asList( + "/vespa/filedistribution", // Path to file distribution data used before Vespa 6.213 + "/vespa/config" // Path to config data used before Vespa 6 + ); + ZooKeeperDataMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { super(applicationRepository, curator, interval); } @Override protected void maintain() { - curator.delete(Path.fromString("/vespa/filedistribution")); - curator.delete(Path.fromString("/vespa/config")); - curator.delete(Path.fromString("/provision/v1/defaultFlavor")); + pathsToDelete.forEach(path -> curator.delete(Path.fromString(path))); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index 70ea20d5b4d..0f9f8b72de1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -134,11 +134,6 @@ public class LocalSession extends Session implements Comparable<LocalSession> { return lhsId.compareTo(rhsId); } - // in seconds - public long getCreateTime() { - return zooKeeperClient.readCreateTime(); - } - public void waitUntilActivated(TimeoutBudget timeoutBudget) { zooKeeperClient.getActiveWaiter().awaitCompletion(timeoutBudget.timeLeft()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index c47d5791a1b..dbccba395a2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -132,4 +132,9 @@ public class RemoteSession extends Session { } } + public void delete() { + Transaction transaction = zooKeeperClient.deleteTransaction(); + transaction.commit(); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index d16d5a17518..8cd4977de65 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import java.time.Duration; +import java.time.Instant; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -48,6 +50,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements Nod private final RemoteSessionFactory remoteSessionFactory; private final Map<Long, RemoteSessionStateWatcher> sessionStateWatchers = new HashMap<>(); private final ReloadHandler reloadHandler; + private final TenantName tenantName; private final MetricUpdater metrics; private final Curator.DirectoryCache directoryCache; private final TenantApplications applicationRepo; @@ -56,20 +59,21 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements Nod * @param curator a {@link Curator} instance. * @param remoteSessionFactory a {@link com.yahoo.vespa.config.server.session.RemoteSessionFactory} * @param reloadHandler a {@link com.yahoo.vespa.config.server.ReloadHandler} - * @param tenant a {@link TenantName} instance. + * @param tenantName a {@link TenantName} instance. * @param applicationRepo a {@link TenantApplications} instance. */ public RemoteSessionRepo(Curator curator, RemoteSessionFactory remoteSessionFactory, ReloadHandler reloadHandler, - TenantName tenant, + TenantName tenantName, TenantApplications applicationRepo, MetricUpdater metricUpdater) { this.curator = curator; - this.sessionsPath = TenantRepository.getSessionsPath(tenant); + this.sessionsPath = TenantRepository.getSessionsPath(tenantName); this.applicationRepo = applicationRepo; this.remoteSessionFactory = remoteSessionFactory; this.reloadHandler = reloadHandler; + this.tenantName = tenantName; this.metrics = metricUpdater; initializeSessions(); this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, pathChildrenExecutor); @@ -82,12 +86,37 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements Nod this.curator = null; this.remoteSessionFactory = null; this.reloadHandler = null; + this.tenantName = tenantName; this.sessionsPath = TenantRepository.getSessionsPath(tenantName); this.metrics = null; this.directoryCache = null; this.applicationRepo = null; } + public List<Long> getSessions() { + return getSessionList(curator.getChildren(sessionsPath)); + } + + public int deleteExpiredSessions(Duration expiryTime, boolean deleteFromZooKeeper) { + int deleted = 0; + for (long sessionId : getSessions()) { + RemoteSession session = getSession(sessionId); + Instant created = Instant.ofEpochSecond(session.getCreateTime()); + if (session.getStatus() == Session.Status.DEACTIVATE && sessionHasExpired(created, expiryTime)) { + log.log(LogLevel.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired"); + if (deleteFromZooKeeper) { + session.delete(); + deleted++; + } + } + } + return deleted; + } + + private boolean sessionHasExpired(Instant created, Duration expiryTime) { + return (created.plus(expiryTime).isBefore(Instant.now())); + } + private void loadActiveSession(RemoteSession session) { tryReload(session.ensureApplicationLoaded(), session.logPre()); } @@ -113,7 +142,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements Nod // TODO: Add sessions in parallel private void initializeSessions() throws NumberFormatException { - getSessionList(curator.getChildren(sessionsPath)).forEach(this::sessionAdded); + getSessions().forEach(this::sessionAdded); } private synchronized void sessionsChanged() throws NumberFormatException { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index e98931b0573..64ecc510fe9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -67,4 +67,9 @@ public abstract class Session { return TenantRepository.logPre(getTenant()); } + // in seconds + public long getCreateTime() { + return zooKeeperClient.readCreateTime(); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index b0320dad88b..6148c276088 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -245,12 +245,11 @@ public class ApplicationRepositoryTest { @Test public void testDeletingInactiveSessions() { ManualClock clock = new ManualClock(Instant.now()); - ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(Files.createTempDir() - .getAbsolutePath()) - .configDefinitionsDir(Files.createTempDir() - .getAbsolutePath()) - .sessionLifetime(60)); + ConfigserverConfig configserverConfig = + new ConfigserverConfig(new ConfigserverConfig.Builder() + .configServerDBDir(Files.createTempDir().getAbsolutePath()) + .configDefinitionsDir(Files.createTempDir().getAbsolutePath()) + .sessionLifetime(60)); DeployTester tester = new DeployTester(configserverConfig, clock); tester.deployApp("src/test/apps/app", "myapp", Instant.now()); // session 2 (numbering starts at 2) @@ -274,11 +273,14 @@ public class ApplicationRepositoryTest { clock.advance(Duration.ofHours(1)); // longer than session lifetime - // All sessions except 3 should be removed after the call to deleteOldSessions - tester.applicationRepository().deleteOldSessions(); + // All sessions except 3 should be removed after the call to deleteExpiredLocalSessions + tester.applicationRepository().deleteExpiredLocalSessions(); final Collection<LocalSession> sessions = tester.tenant().getLocalSessionRepo().listSessions(); assertEquals(1, sessions.size()); assertEquals(3, new ArrayList<>(sessions).get(0).getSessionId()); + + // There should be no expired remote sessions in the common case + assertEquals(0, applicationRepository.deleteExpiredRemoteSessions(Duration.ofSeconds(0))); } private PrepareResult prepareAndActivateApp(File application) throws IOException { diff --git a/container-disc/pom.xml b/container-disc/pom.xml index c112466bbd4..ea3425d302d 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -169,6 +169,7 @@ configdefinitions-jar-with-dependencies.jar, container-jersey2-jar-with-dependencies.jar, container-search-and-docproc-jar-with-dependencies.jar, + container-search-gui-jar-with-dependencies.jar, docprocs-jar-with-dependencies.jar, jdisc-security-filters-jar-with-dependencies.jar, jdisc_http_service-jar-with-dependencies.jar, diff --git a/container-integration-test/.gitignore b/container-integration-test/.gitignore new file mode 100644 index 00000000000..3cc25b51fc4 --- /dev/null +++ b/container-integration-test/.gitignore @@ -0,0 +1,2 @@ +/pom.xml.build +/target diff --git a/container-integration-test/OWNERS b/container-integration-test/OWNERS new file mode 100644 index 00000000000..3b2ba1ede81 --- /dev/null +++ b/container-integration-test/OWNERS @@ -0,0 +1 @@ +gjoranv diff --git a/container-integration-test/README.md b/container-integration-test/README.md new file mode 100644 index 00000000000..8abe3a889ea --- /dev/null +++ b/container-integration-test/README.md @@ -0,0 +1,9 @@ +<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +# Integration tests for JDisc components + +This module contains integration tests for container components. + +Tests that use the `application` framework cannot be added to the same maven +module as the component itself because that will usually create a cycle in the +dependency graph. diff --git a/container-integration-test/pom.xml b/container-integration-test/pom.xml new file mode 100644 index 00000000000..4b4d5de21eb --- /dev/null +++ b/container-integration-test/pom.xml @@ -0,0 +1,44 @@ +<?xml version="1.0"?> +<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>container-integration-test</artifactId> + <packaging>jar</packaging> + <version>6-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-search-gui</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>application</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + </plugins> + </build> +</project>
\ No newline at end of file diff --git a/container-search-gui/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java index ec515a1bf4a..5fd73afe800 100644 --- a/container-search-gui/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java +++ b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java @@ -5,13 +5,12 @@ import com.yahoo.application.Networking; import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; + import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; - import java.io.IOException; - +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -25,7 +24,7 @@ public class GUIHandlerTest { @Before public void startContainer() { - container = JDisc.fromServicesXml(servicesXml(), Networking.enable); + container = JDisc.fromServicesXml(servicesXml(), Networking.disable); } @After @@ -64,8 +63,8 @@ public class GUIHandlerTest { private void assertResponse(Request.Method method, String path, String expectedStartString, String expectedContentType, int expectedStatusCode) throws IOException { Response response = container.handleRequest(new Request("http://localhost:8080" + path, new byte[0], method)); - Assert.assertEquals("Status code", expectedStatusCode, response.getStatus()); - Assert.assertEquals(expectedContentType, response.getHeaders().getFirst("Content-Type")); + assertEquals("Status code", expectedStatusCode, response.getStatus()); + assertEquals(expectedContentType, response.getHeaders().getFirst("Content-Type")); if(expectedStartString != null){ assertTrue(response.getBodyAsString().startsWith(expectedStartString)); } diff --git a/container-search-gui/.gitignore b/container-search-gui/.gitignore index 7f4ebc3a7c6..3cc25b51fc4 100644 --- a/container-search-gui/.gitignore +++ b/container-search-gui/.gitignore @@ -1,31 +1,2 @@ -.classpath -.project -.settings -/.cache -/.classpath -/.emacs.desktop -/.nbintdb -/.project -/.settings -/.version -/QueryAccessLog* -/accessLog.log -/build -/bundles -/libexec -/nbproject /pom.xml.build -/prelude.iml -/prelude.ipr -/prelude.iws -/staging /target -/temp -/testLogFileG.txt -/testLogs -/test_yapache_access_log -/testlogsG -/testng.out.log -/tmp -null.log -tmp diff --git a/container-search-gui/CMakeLists.txt b/container-search-gui/CMakeLists.txt new file mode 100644 index 00000000000..95476154443 --- /dev/null +++ b/container-search-gui/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(container-search-gui)
\ No newline at end of file diff --git a/container-search-gui/pom.xml b/container-search-gui/pom.xml index f2ca70dd1f3..f598758e1de 100644 --- a/container-search-gui/pom.xml +++ b/container-search-gui/pom.xml @@ -4,46 +4,55 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>container-search-gui</artifactId> + <packaging>container-plugin</packaging> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>container-search-gui</artifactId> - <packaging>jar</packaging> - <version>6-SNAPSHOT</version> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>application</artifactId> - <version>6-SNAPSHOT</version> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Xlint:-rawtypes</arg> - <arg>-Xlint:-unchecked</arg> - <arg>-Xlint:-serial</arg> - <arg>-Xlint:-deprecation</arg> - <arg>-Xlint:-dep-ann</arg> - <arg>-Xlint:-cast</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - </plugins> - </build> + <dependencies> + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>yolean</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_http_service</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + </plugins> + </build> + </project>
\ No newline at end of file diff --git a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java index 0f4eebd7fd7..95adbbea3fc 100644 --- a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java +++ b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java @@ -5,17 +5,13 @@ import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; - - import com.yahoo.search.query.restapi.ErrorResponse; import com.yahoo.yolean.Exceptions; -import java.io.File; + import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.logging.Level; /** @@ -47,75 +43,83 @@ public class GUIHandler extends LoggingRequestHandler { } private HttpResponse handleGET(HttpRequest request) { - com.yahoo.restapi.Path path = new com.yahoo.restapi.Path(request.getUri().getPath()); + com.yahoo.restapi.Path path = new com.yahoo.restapi.Path(request.getUri().getPath()); if (path.matches("/querybuilder/")) { return new FileResponse("_includes/index.html"); } if (!path.matches("/querybuilder/{*}") ) { - return ErrorResponse.notFoundError("Nothing at " + path); + return ErrorResponse.notFoundError("Nothing at path:" + path); } String filepath = path.getRest(); - if (!isValidPath(GUIHandler.class.getClassLoader().getResource("gui").getFile()+"/"+filepath)){ - return ErrorResponse.notFoundError("Nothing at " + path); + if (!isValidPath(filepath)){ + return ErrorResponse.notFoundError("Nothing at path:" + filepath); } return new FileResponse(filepath); } private static boolean isValidPath(String path) { - File file = new File(path); - return file.exists(); + InputStream in = GUIHandler.class.getClassLoader().getResourceAsStream("gui/"+path); + boolean isValid = (in != null); + if(isValid){ + try { in.close(); } catch (IOException e) {/* Problem with closing inputstream */} + } + + return isValid; } private static class FileResponse extends HttpResponse { - private final Path path; + private final String path; public FileResponse(String relativePath) { super(200); - this.path = Paths.get(GUIHandler.class.getClassLoader().getResource("gui").getFile(), relativePath); + this.path = relativePath; } @Override public void render(OutputStream out) throws IOException { - byte[] data = Files.readAllBytes(path); - out.write(data); + InputStream is = GUIHandler.class.getClassLoader().getResourceAsStream("gui/"+this.path); + byte[] buf = new byte[1024]; + int numRead; + while ( (numRead = is.read(buf) ) >= 0) { + out.write(buf, 0, numRead); + } } @Override public String getContentType() { - if (path.toString().endsWith(".css")) { + if (path.endsWith(".css")) { return "text/css"; - } else if (path.toString().endsWith(".js")) { + } else if (path.endsWith(".js")) { return "application/javascript"; - } else if (path.toString().endsWith(".html")) { + } else if (path.endsWith(".html")) { return "text/html"; - }else if (path.toString().endsWith(".php")) { + } else if (path.endsWith(".php")) { return "text/php"; - }else if (path.toString().endsWith(".svg")) { + } else if (path.endsWith(".svg")) { return "image/svg+xml"; - }else if (path.toString().endsWith(".eot")) { + } else if (path.endsWith(".eot")) { return "application/vnd.ms-fontobject"; - }else if (path.toString().endsWith(".ttf")) { + } else if (path.endsWith(".ttf")) { return "font/ttf"; - }else if (path.toString().endsWith(".woff")) { + } else if (path.endsWith(".woff")) { return "font/woff"; - }else if (path.toString().endsWith(".woff2")) { + } else if (path.endsWith(".woff2")) { return "font/woff2"; - }else if (path.toString().endsWith(".otf")) { + } else if (path.endsWith(".otf")) { return "font/otf"; - }else if (path.toString().endsWith(".png")) { + } else if (path.endsWith(".png")) { return "image/png"; - }else if (path.toString().endsWith(".xml")) { + } else if (path.endsWith(".xml")) { return "application/xml"; - }else if (path.toString().endsWith(".ico")) { + } else if (path.endsWith(".ico")) { return "image/x-icon"; - }else if (path.toString().endsWith(".json")) { + } else if (path.endsWith(".json")) { return "application/json"; - }else if (path.toString().endsWith(".ttf")) { + } else if (path.endsWith(".ttf")) { return "font/ttf"; } return "text/html"; } } - -} +}
\ No newline at end of file @@ -56,6 +56,7 @@ <module>container-dev</module> <module>container-di</module> <module>container-disc</module> + <module>container-integration-test</module> <module>container-jersey2</module> <module>container-messagebus</module> <module>container-search-and-docproc</module> diff --git a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h index c432606f7f0..08bb3153838 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h +++ b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h @@ -2,6 +2,7 @@ #pragma once +#include <vector> #include "collectiontype.h" #include "basictype.h" #include <vespa/searchcommon/common/iblobconverter.h> @@ -251,6 +252,16 @@ public: virtual bool findEnum(const char * value, EnumHandle & e) const = 0; /** + * Finds all enum values matching the given string value. + * This method will only have effect if @ref getBasicType() returns BasicType::STRING and + * @ref hasEnum() returns true. + * + * @param value the string value to lookup. + * @return vector of EnumHandles, size 0 if no match found. + **/ + virtual std::vector<EnumHandle> findFoldedEnums(const char * value) const = 0; + + /** * Given an enum handle, returns the string it refers to. * This method will only have effect if @ref getBasicType() returns BasicType::STRING and * @ref hasEnum() returns true. @@ -427,6 +438,6 @@ private: }; -} // namespace fef +} // namespace attribute } // namespace search diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 5148ab2be34..32ac836302a 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -1297,6 +1297,9 @@ AttributeTest::testWeightedSet() testWeightedSet<IntegerAttribute, AttributeVector::WeightedInt>(ptr, values); IAttributeVector::EnumHandle e; EXPECT_TRUE(ptr->findEnum("1", e)); + EXPECT_EQUAL(1u, ptr->findFoldedEnums("1").size()); + EXPECT_EQUAL(e, ptr->findFoldedEnums("1")[0]); + } } { // FloatingPointAttribute @@ -1320,6 +1323,8 @@ AttributeTest::testWeightedSet() testWeightedSet<FloatingPointAttribute, AttributeVector::WeightedFloat>(ptr, values); IAttributeVector::EnumHandle e; EXPECT_TRUE(ptr->findEnum("1", e)); + EXPECT_EQUAL(1u, ptr->findFoldedEnums("1").size()); + EXPECT_EQUAL(e, ptr->findFoldedEnums("1")[0]); } } { // StringAttribute @@ -1345,6 +1350,8 @@ AttributeTest::testWeightedSet() testWeightedSet<StringAttribute, AttributeVector::WeightedString>(ptr, values); IAttributeVector::EnumHandle e; EXPECT_TRUE(ptr->findEnum("string00", e)); + EXPECT_EQUAL(1u, ptr->findFoldedEnums("StRiNg00").size()); + EXPECT_EQUAL(e, ptr->findFoldedEnums("StRiNg00")[0]); } } } diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index daff432d68d..89dd1cfdab4 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -41,6 +41,7 @@ private: void testFloatEnumStore(EnumStoreType & es); void testFloatEnumStore(); + void testFindFolded(); void testAddEnum(); template <typename EnumStoreType> void testAddEnum(bool hasPostings); @@ -275,6 +276,40 @@ EnumStoreTest::testFloatEnumStore() } void +EnumStoreTest::testFindFolded() +{ + StringEnumStore ses(100, false); + std::vector<EnumIndex> indices; + std::vector<std::string> unique({"", "one", "two", "TWO", "Two", "three"}); + for (std::string &str : unique) { + EnumIndex idx; + ses.addEnum(str.c_str(), idx); + EXPECT_TRUE(ses.getLastEnum() == indices.size()); + indices.push_back(idx); + ses.incRefCount(idx); + EXPECT_EQUAL(1u, ses.getRefCount(idx)); + } + ses.freezeTree(); + for (uint32_t i = 0; i < indices.size(); ++i) { + uint32_t e = ses.getEnum(indices[i]); + EXPECT_EQUAL(i, e); + EnumIndex idx; + EXPECT_TRUE(ses.findIndex(unique[i].c_str(), idx)); + } + EXPECT_EQUAL(1u, ses.findFoldedEnums("").size()); + EXPECT_EQUAL(0u, ses.findFoldedEnums("foo").size()); + EXPECT_EQUAL(1u, ses.findFoldedEnums("one").size()); + EXPECT_EQUAL(3u, ses.findFoldedEnums("two").size()); + EXPECT_EQUAL(3u, ses.findFoldedEnums("TWO").size()); + EXPECT_EQUAL(3u, ses.findFoldedEnums("tWo").size()); + const auto v = ses.findFoldedEnums("Two"); + EXPECT_EQUAL(std::string("TWO"), ses.getValue(v[0])); + EXPECT_EQUAL(std::string("Two"), ses.getValue(v[1])); + EXPECT_EQUAL(std::string("two"), ses.getValue(v[2])); + EXPECT_EQUAL(1u, ses.findFoldedEnums("three").size()); +} + +void EnumStoreTest::testAddEnum() { testAddEnum<StringEnumStore>(false); @@ -319,6 +354,8 @@ EnumStoreTest::testAddEnum(bool hasPostings) uint32_t e = ses.getEnum(indices[i]); EXPECT_EQUAL(i, e); EXPECT_TRUE(ses.findEnum(unique[i].c_str(), e)); + EXPECT_EQUAL(1u, ses.findFoldedEnums(unique[i].c_str()).size()); + EXPECT_EQUAL(e, ses.findFoldedEnums(unique[i].c_str())[0]); EXPECT_TRUE(ses.getEnum(datastore::EntryRef(e)) == i); EXPECT_TRUE(ses.findIndex(unique[i].c_str(), idx)); EXPECT_TRUE(idx == indices[i]); @@ -874,6 +911,7 @@ EnumStoreTest::Main() testStringEntry(); testNumericEntry(); testFloatEnumStore(); + testFindFolded(); testAddEnum(); testCompaction(); testReset(); diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index b25abdfd2e5..2adfdd135df 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -153,6 +153,8 @@ StringAttributeTest::testMultiValue(Attribute & attr, uint32_t numDocs) EXPECT_TRUE(strcmp(attr.get(doc), uniqueStrings[0].c_str()) == 0); uint32_t e; EXPECT_TRUE(attr.findEnum(uniqueStrings[0].c_str(), e)); + EXPECT_EQUAL(1u, attr.findFoldedEnums(uniqueStrings[0].c_str()).size()); + EXPECT_EQUAL(e, attr.findFoldedEnums(uniqueStrings[0].c_str())[0]); EXPECT_TRUE(attr.getEnum(doc) == e); } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp index 3ff7db5a184..2ad1cc0d739 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp @@ -36,14 +36,8 @@ class UseStringEnum : public UseAttr public: UseStringEnum(const IAttributeVector & attr) : UseAttr(attr) {} - bool mapToken(const ISearchContext &context, int64_t &token) const - { - attribute::IAttributeVector::EnumHandle handle; - if (attribute().findEnum(context.queryTerm().getTerm(), handle)) { - token = handle; - return true; - } - return false; + auto mapToken(const ISearchContext &context) const { + return attribute().findFoldedEnums(context.queryTerm().getTerm()); } int64_t getToken(uint32_t docId) const { return attribute().getEnum(docId); @@ -56,14 +50,13 @@ class UseInteger : public UseAttr { public: UseInteger(const IAttributeVector & attr) : UseAttr(attr) {} - bool mapToken(const ISearchContext &context, int64_t &token) const - { + std::vector<int64_t> mapToken(const ISearchContext &context) const { + std::vector<int64_t> result; Int64Range range(context.getAsIntegerTerm()); if (range.isPoint()) { - token = range.lower(); - return true; + result.push_back(range.lower()); } - return false; + return result; } int64_t getToken(uint32_t docId) const { return attribute().getInt(docId); @@ -92,8 +85,7 @@ public: : _tfmd(tfmd), _attr(attr), _map(), _weight(0) { for (size_t i = 0; i < contexts.size(); ++i) { - int64_t token(0); - if (_attr.mapToken(*contexts[i], token)) { + for (int64_t token : _attr.mapToken(*contexts[i])) { _map[token] = weights[i]; } } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 8b90677a1bc..fbad06821cd 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -508,6 +508,12 @@ int32_t AttributeVector::getWeight(DocId, uint32_t) const { return 1; } bool AttributeVector::findEnum(const char *, EnumHandle &) const { return false; } +std::vector<search::attribute::IAttributeVector::EnumHandle> +AttributeVector::findFoldedEnums(const char *) const { + std::vector<EnumHandle> empty; + return empty; +} + const char * AttributeVector::getStringFromEnum(EnumHandle) const { return nullptr; } AttributeVector::SearchContext::SearchContext(const AttributeVector &attr) : diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 87ef9a41432..1e249845ed2 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -500,6 +500,8 @@ public: // Implements IAttributeVector bool findEnum(const char *value, EnumHandle &e) const override; + std::vector<EnumHandle> findFoldedEnums(const char *) const override; + const char * getStringFromEnum(EnumHandle e) const override; ///// Modify API diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp index 94b847a02e6..41945dff1fe 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp @@ -51,6 +51,19 @@ bool StringDirectAttribute::findEnum(const char * key, EnumHandle & e) const return false; } + +// XXX this is not really correct +std::vector<StringDirectAttribute::EnumHandle> +StringDirectAttribute::findFoldedEnums(const char *key) const +{ + std::vector<EnumHandle> result; + EnumHandle handle; + if (findEnum(key, handle)) { + result.push_back(handle); + } + return result; +} + void StringDirectAttribute::onSave(IAttributeSaveTarget & saveTarget) { assert(!saveTarget.getEnumerated()); diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.h b/searchlib/src/vespa/searchlib/attribute/attrvector.h index a6f9a0ebcee..2ba9ed083f0 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.h +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.h @@ -152,6 +152,7 @@ protected: StringDirectAttribute(const vespalib::string & baseFileName, const Config & c); ~StringDirectAttribute(); bool findEnum(const char * value, EnumHandle & e) const override; + std::vector<EnumHandle> findFoldedEnums(const char *) const override; void getEnumValue(const EnumHandle * v, uint32_t *e, uint32_t sz) const override { for (size_t i(0); i < sz; i++) { e[i] = v[i]; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index c1f5d278827..3206ea62d73 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -166,6 +166,7 @@ public: ssize_t deserialize(const void *src, size_t available, Index &idx) override; bool foldedChange(const Index &idx1, const Index &idx2) override; virtual bool findEnum(Type value, EnumStoreBase::EnumHandle &e) const; + virtual std::vector<EnumStoreBase::EnumHandle> findFoldedEnums(Type value) const; void addEnum(Type value, Index &newIdx); virtual bool findIndex(Type value, Index &idx) const; void freeUnusedEnums(bool movePostingidx) override; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 54100bafa1e..794138c7c89 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -200,6 +200,15 @@ EnumStoreT<EntryType>::findEnum(Type value, } template <typename EntryType> +std::vector<EnumStoreBase::EnumHandle> +EnumStoreT<EntryType>::findFoldedEnums(Type value) const +{ + FoldedComparatorType cmp(*this, value); + return _enumDict->findMatchingEnums(cmp); +} + + +template <typename EntryType> bool EnumStoreT<EntryType>::findIndex(Type value, Index &idx) const { diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp index 914a56e7abc..61d862b6c4f 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp @@ -507,6 +507,20 @@ EnumStoreDict<Dictionary>::findFrozenIndex(const EnumStoreComparator &cmp, template <typename Dictionary> +std::vector<EnumStoreBase::EnumHandle> +EnumStoreDict<Dictionary>::findMatchingEnums(const EnumStoreComparator &cmp) const +{ + std::vector<EnumStoreBase::EnumHandle> result; + typename Dictionary::ConstIterator itr = + _dict.getFrozenView().find(Index(), cmp); + while (itr.valid() && !cmp(Index(), itr.getKey())) { + result.push_back(itr.getKey().ref()); + ++itr; + } + return result; +} + +template <typename Dictionary> void EnumStoreDict<Dictionary>::onReset() { diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h index f74345a8806..9bea2a568e1 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h @@ -78,6 +78,9 @@ public: const EnumStoreComparator *fcmp) = 0; virtual bool findIndex(const EnumStoreComparator &cmp, Index &idx) const = 0; virtual bool findFrozenIndex(const EnumStoreComparator &cmp, Index &idx) const = 0; + virtual std::vector<attribute::IAttributeVector::EnumHandle> + findMatchingEnums(const EnumStoreComparator &cmp) const = 0; + virtual void onReset() = 0; virtual void onTransferHoldLists(generation_t generation) = 0; virtual void onTrimHoldLists(generation_t firstUsed) = 0; @@ -131,6 +134,9 @@ public: bool findIndex(const EnumStoreComparator &cmp, Index &idx) const override; bool findFrozenIndex(const EnumStoreComparator &cmp, Index &idx) const override; + std::vector<attribute::IAttributeVector::EnumHandle> + findMatchingEnums(const EnumStoreComparator &cmp) const override; + void onReset() override; void onTransferHoldLists(generation_t generation) override; void onTrimHoldLists(generation_t firstUsed) override; diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.h b/searchlib/src/vespa/searchlib/attribute/floatbase.h index df156fd9fc5..955b2b252af 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.h +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.h @@ -81,6 +81,7 @@ protected: Change _defaultValue; private: bool findEnum(const char *value, EnumHandle &e) const override; + std::vector<EnumHandle> findFoldedEnums(const char *value) const override; bool isUndefined(DocId doc) const override; virtual T get(DocId doc) const = 0; virtual T getFromEnum(EnumHandle e) const = 0; diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.hpp b/searchlib/src/vespa/searchlib/attribute/floatbase.hpp index 669c7974e6f..a454fd3e235 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.hpp +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.hpp @@ -48,6 +48,18 @@ FloatingPointAttributeTemplate<T>::findEnum(const char *value, EnumHandle &e) co } template<typename T> +std::vector<EnumStoreBase::EnumHandle> +FloatingPointAttributeTemplate<T>::findFoldedEnums(const char *value) const +{ + std::vector<EnumHandle> result; + EnumHandle h; + if (findEnum(value, h)) { + result.push_back(h); + } + return result; +} + +template<typename T> bool FloatingPointAttributeTemplate<T>::isUndefined(DocId doc) const { return attribute::isUndefined(get(doc)); diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index e6dacde3600..428b14671cd 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -99,6 +99,11 @@ bool ImportedAttributeVectorReadGuard::findEnum(const char *value, EnumHandle &e return _target_attribute.findEnum(value, e); } +std::vector<ImportedAttributeVectorReadGuard::EnumHandle> +ImportedAttributeVectorReadGuard::findFoldedEnums(const char *value) const { + return _target_attribute.findFoldedEnums(value); +} + const char * ImportedAttributeVectorReadGuard::getStringFromEnum(EnumHandle e) const { return _target_attribute.getStringFromEnum(e); } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index 2622f091cf1..4cf4d5b64c1 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -67,6 +67,8 @@ public: virtual uint32_t get(DocId docId, WeightedConstChar *buffer, uint32_t sz) const override; virtual uint32_t get(DocId docId, WeightedEnum *buffer, uint32_t sz) const override; virtual bool findEnum(const char * value, EnumHandle & e) const override; + virtual std::vector<EnumHandle> findFoldedEnums(const char *value) const override; + virtual const char * getStringFromEnum(EnumHandle e) const override; virtual std::unique_ptr<ISearchContext> createSearchContext(std::unique_ptr<QueryTermSimple> term, const SearchContextParams ¶ms) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.h b/searchlib/src/vespa/searchlib/attribute/integerbase.h index 8d6c5046070..5dec40fd4da 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.h @@ -98,6 +98,8 @@ protected: Change _defaultValue; private: bool findEnum(const char *value, EnumHandle &e) const override; + std::vector<EnumHandle> findFoldedEnums(const char *value) const override; + virtual T get(DocId doc) const = 0; virtual T getFromEnum(EnumHandle e) const = 0; largeint_t getIntFromEnum(EnumHandle e) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.hpp b/searchlib/src/vespa/searchlib/attribute/integerbase.hpp index 3866e5a4acb..7b23cd51e92 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.hpp +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.hpp @@ -32,6 +32,19 @@ IntegerAttributeTemplate<T>::findEnum(const char *value, EnumHandle &e) const { return findEnum(ivalue, e); } + +template<typename T> +std::vector<EnumStoreBase::EnumHandle> +IntegerAttributeTemplate<T>::findFoldedEnums(const char *value) const +{ + std::vector<EnumHandle> result; + EnumHandle h; + if (findEnum(value, h)) { + result.push_back(h); + } + return result; +} + template<typename T> largeint_t IntegerAttributeTemplate<T>::getIntFromEnum(EnumHandle e) const { diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h index 5e5d33419aa..eb6a8b630de 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h @@ -71,6 +71,11 @@ public: return this->_enumStore.getValue(indices[0].value()); } } + + std::vector<EnumHandle> findFoldedEnums(const char *value) const override { + return this->_enumStore.findFoldedEnums(value); + } + const char * getStringFromEnum(EnumHandle e) const override { return this->_enumStore.getValue(e); } diff --git a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp index 5e6cfa8a847..1dc95c42de8 100644 --- a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp @@ -103,6 +103,12 @@ NotImplementedAttribute::findEnum(const char *, EnumHandle &) const { return false; } +std::vector<NotImplementedAttribute::EnumHandle> +NotImplementedAttribute::findFoldedEnums(const char *) const { + notImplemented(); + return std::vector<EnumHandle>(); +} + long NotImplementedAttribute::onSerializeForAscendingSort(DocId, void *, long, const common::BlobConverter *) const { notImplemented(); diff --git a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h index 98935c1f155..cbd2ff162b2 100644 --- a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h @@ -26,6 +26,8 @@ struct NotImplementedAttribute : AttributeVector { uint32_t get(DocId, WeightedConstChar *, uint32_t) const override; uint32_t get(DocId, WeightedEnum *, uint32_t) const override; bool findEnum(const char *, EnumHandle &) const override; + std::vector<EnumHandle> findFoldedEnums(const char *value) const override; + long onSerializeForAscendingSort(DocId, void *, long, const common::BlobConverter *) const override; long onSerializeForDescendingSort(DocId, void *, long, const common::BlobConverter *) const override; uint32_t clearDoc(DocId) override; diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h index 4993b295b37..8d2efcbdc09 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h @@ -52,6 +52,9 @@ public: const char * get(DocId doc) const override { return this->_enumStore.getValue(this->_enumIndices[doc]); } + std::vector<EnumHandle> findFoldedEnums(const char *value) const override { + return this->_enumStore.findFoldedEnums(value); + } const char * getStringFromEnum(EnumHandle e) const override { return this->_enumStore.getValue(e); } diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h index c817332af15..279acce534d 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.h +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h @@ -44,6 +44,7 @@ public: bool apply(DocId doc, const ArithmeticValueUpdate & op); bool applyWeight(DocId doc, const FieldValue & fv, const ArithmeticValueUpdate & wAdjust) override; bool findEnum(const char * value, EnumHandle & e) const override = 0; + std::vector<EnumHandle> findFoldedEnums(const char *value) const override = 0; uint32_t get(DocId doc, largeint_t * v, uint32_t sz) const override; uint32_t get(DocId doc, double * v, uint32_t sz) const override; uint32_t get(DocId doc, WeightedInt * v, uint32_t sz) const override; diff --git a/standalone-container/vespa-standalone-container.spec b/standalone-container/vespa-standalone-container.spec index 05a126e49b3..281516ae552 100644 --- a/standalone-container/vespa-standalone-container.spec +++ b/standalone-container/vespa-standalone-container.spec @@ -48,6 +48,7 @@ declare -a modules=( container-disc container-jersey2 container-search-and-docproc + container-search-gui defaults docprocs jdisc-security-filters diff --git a/vagrant/README.md b/vagrant/README.md index d81e80e8808..d5212692638 100644 --- a/vagrant/README.md +++ b/vagrant/README.md @@ -1,6 +1,6 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# Create C++ dev environment on CentOS using VirtualBox and Vagrant +# Create C++ / Java dev environment on CentOS using VirtualBox and Vagrant ## Prerequisites * [Install VirtualBox](https://www.virtualbox.org/wiki/Downloads) @@ -12,16 +12,41 @@ cd <vespa-source>/vagrant -#### 2. Install Vagrant VirtualBox Guest Additions plugin +#### 2. Choose dev environment + +##### a. For a dev environment with plain centos/7 and no GUI: + + export VESPA_VAGRANT_VM_BOX=centos/7 + export VESPA_VAGRANT_DISABLE_GUI=true + +##### b. For a dev environment with GUI and CLion: + +Create centos7-desktop box: + +* Install packer by following guide at [packer.io](https://www.packer.io/intro/getting-started/install.html) + +* Clone boxcutter centos repo and build the box: +``` +git clone https://github.com/boxcutter/centos.git +./bin/box build centos7-desktop.json virtualbox +``` + +Example exports: + + export VESPA_VAGRANT_VM_BOX="centos7-desktop" + export VESPA_VAGRANT_VM_BOX_URL="$HOME/git/boxcutter/centos/box/virtualbox/centos7-desktop-xx.yyyy.z.box" + + +#### 3. Install Vagrant VirtualBox Guest Additions plugin This is required for mounting shared folders and get mouse pointer integration and seamless windows in the virtual CentOS desktop. vagrant plugin install vagrant-vbguest -#### 3. Start and provision the environment +#### 4. Start and provision the environment vagrant up -#### 4. Connect to machine via SSH +#### 5. Connect to machine via SSH SSH agent forwarding is enabled to ensure easy interaction with GitHub inside the machine. vagrant ssh @@ -31,6 +56,10 @@ This is needed in order to compile and run tests fast on the local file system i git clone git@github.com:vespa-engine/vespa.git +## Build Java modules +Please follow the build instructions described [here](../README.md#build-java-modules). + + ## Build C++ modules Please follow the build instructions described [here](../README.md#build-c-modules). Skip these steps if doing development with CLion. diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 84836d05e72..f15f45d75a0 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -1,25 +1,37 @@ # -*- mode: ruby -*- # vi: set ft=ruby : +disable_gui = ENV['VESPA_VAGRANT_DISABLE_GUI'] -def validate_vm_env_option(name) +def get_mandatory_env_value(name) opt = ENV[name] if opt.nil? or opt.empty? raise Vagrant::Errors::VagrantError.new, "Environment variable #{name} must be set to a valid value before running vagrant" end + return opt end -validate_vm_env_option('VESPA_VAGRANT_VM_BOX') -validate_vm_env_option('VESPA_VAGRANT_VM_BOX_URL') +def get_env_value(name, fallback) + opt = ENV[name] + if opt.nil? or opt.empty? + return fallback + end + return opt +end -vm_box = ENV['VESPA_VAGRANT_VM_BOX'] -vm_box_url = ENV['VESPA_VAGRANT_VM_BOX_URL'] +vm_box = get_mandatory_env_value('VESPA_VAGRANT_VM_BOX') +vm_memory = get_env_value('VESPA_VAGRANT_VM_MEMORY', "8192") +vm_cpus = get_env_value('VESPA_VAGRANT_VM_CPUS', 4) + +unless disable_gui + vm_box_url = get_mandatory_env_value('VESPA_VAGRANT_VM_BOX_URL') +end # For a complete reference, please see the online documentation at https://docs.vagrantup.com. Vagrant.configure("2") do |config| config.vm.box = vm_box - config.vm.box_url = vm_box_url + config.vm.box_url = vm_box_url unless disable_gui config.ssh.forward_agent = true @@ -27,11 +39,11 @@ Vagrant.configure("2") do |config| config.vm.provider "virtualbox" do |vb| # Display the VirtualBox GUI when booting the machine - vb.gui = true + vb.gui = true unless disable_gui vb.name = "vespa-dev" - vb.memory = "8192" - vb.cpus = 4 + vb.memory = vm_memory + vb.cpus = vm_cpus end # Install required and nice-to-have packages @@ -52,9 +64,13 @@ Vagrant.configure("2") do |config| yum-builddep -y /vagrant/dist/vespa.spec echo -e "* soft nproc 409600\n* hard nproc 409600" > /etc/security/limits.d/99-nproc.conf echo -e "* soft nofile 262144\n* hard nofile 262144" > /etc/security/limits.d/99-nofile.conf - echo -e "fs.inotify.max_user_watches = 524288" > /etc/sysctl.d/clion.conf - wget -q -O - https://download.jetbrains.com/cpp/CLion-2017.3.3.tar.gz | tar -C /opt -zx - ln -sf /opt/clion-2017.3.3/bin/clion.sh /usr/bin/clion + + unless disable_gui + echo -e "fs.inotify.max_user_watches = 524288" > /etc/sysctl.d/clion.conf + wget -q -O - https://download.jetbrains.com/cpp/CLion-2018.1.6.tar.gz | tar -C /opt -zx + ln -sf /opt/clion-2018.1.6/bin/clion.sh /usr/bin/clion + end + yum update -y hostname localhost SHELL |