diff options
180 files changed, 2466 insertions, 1690 deletions
diff --git a/application/src/test/java/com/yahoo/application/ApplicationTest.java b/application/src/test/java/com/yahoo/application/ApplicationTest.java index 9388c8f400e..7b4f39b6ac4 100644 --- a/application/src/test/java/com/yahoo/application/ApplicationTest.java +++ b/application/src/test/java/com/yahoo/application/ApplicationTest.java @@ -372,4 +372,21 @@ public class ApplicationTest { "</jdisc>"; } + @Test + public void application_with_access_control_can_be_constructed() throws Exception { + try (Application application = Application.fromServicesXml(servicesXmlWithAccessControl(), Networking.disable)) { + Application unused = application; + } + } + + private static String servicesXmlWithAccessControl() { + return "<jdisc version='1.0'>" + + " <http> <server port='" + 0 +"' id='foo'/> " + + " <filtering>" + + " <access-control domain='foo' />" + + " </filtering>" + + " </http>" + + "</jdisc>"; + } + } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java index e7e65751ee8..783f7361ad5 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java @@ -56,20 +56,23 @@ public class SchemaValidators { */ public SchemaValidators(Version vespaVersion, DeployLogger logger) { this.deployLogger = logger; - File schemaDir; + File schemaDir = null; try { schemaDir = saveSchemasFromJar(new File(SchemaValidators.schemaDirBase), vespaVersion); - } catch (IOException e) { - throw new RuntimeException(e); + servicesXmlValidator = createValidator(schemaDir, servicesXmlSchemaName); + hostsXmlValidator = createValidator(schemaDir, hostsXmlSchemaName); + deploymentXmlValidator = createValidator(schemaDir, deploymentXmlSchemaName); + validationOverridesXmlValidator = createValidator(schemaDir, validationOverridesXmlSchemaName); + containerIncludeXmlValidator = createValidator(schemaDir, containerIncludeXmlSchemaName); + routingStandaloneXmlValidator = createValidator(schemaDir, routingStandaloneXmlSchemaName); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } catch (Exception e) { + throw e; + } finally { + if (schemaDir != null) + IOUtils.recursiveDeleteDir(schemaDir); } - - servicesXmlValidator = createValidator(schemaDir, servicesXmlSchemaName); - hostsXmlValidator = createValidator(schemaDir, hostsXmlSchemaName); - deploymentXmlValidator = createValidator(schemaDir, deploymentXmlSchemaName); - validationOverridesXmlValidator = createValidator(schemaDir, validationOverridesXmlSchemaName); - containerIncludeXmlValidator = createValidator(schemaDir, containerIncludeXmlSchemaName); - routingStandaloneXmlValidator = createValidator(schemaDir, routingStandaloneXmlSchemaName); - IOUtils.recursiveDeleteDir(schemaDir); } /** @@ -81,19 +84,19 @@ public class SchemaValidators { this(vespaVersion, new BaseDeployLogger()); } - public SchemaValidator servicesXmlValidator() throws IOException { + public SchemaValidator servicesXmlValidator() { return servicesXmlValidator; } - public SchemaValidator hostsXmlValidator() throws IOException { + public SchemaValidator hostsXmlValidator() { return hostsXmlValidator; } - public SchemaValidator deploymentXmlValidator() throws IOException { + public SchemaValidator deploymentXmlValidator() { return deploymentXmlValidator; } - SchemaValidator validationOverridesXmlValidator() throws IOException { + SchemaValidator validationOverridesXmlValidator() { return validationOverridesXmlValidator; } diff --git a/config-model/pom.xml b/config-model/pom.xml index 9388608fdcb..bab2f37e667 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -276,11 +276,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.major-version}</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> </dependency> @@ -300,11 +295,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-xml_${scala.major-version}</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> </dependency> @@ -531,19 +521,6 @@ <updateReleaseInfo>true</updateReleaseInfo> </configuration> </plugin> - <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <id>test-compile</id> - <goals> - <goal>testCompile</goal> - </goals> - <phase>test-compile</phase> - </execution> - </executions> - </plugin> </plugins> </build> </project> diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 721e1c08989..7f8ff6edd85 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -48,8 +48,9 @@ public class RankSetupValidator extends Validator { @Override public void validate(VespaModel model, DeployState deployState) { + File cfgDir = null; try { - File cfgDir = Files.createTempDirectory("deploy_ranksetup").toFile(); + cfgDir = Files.createTempDirectory("deploy_ranksetup").toFile(); for (AbstractSearchCluster cluster : model.getSearchClusters()) { // Skipping rank expression checking for streaming clusters, not implemented yet @@ -66,9 +67,12 @@ public class RankSetupValidator extends Validator { } } } - deleteTempDir(cfgDir); + } catch (IOException e) { throw new RuntimeException(e); + } finally { + if (cfgDir != null) + deleteTempDir(cfgDir); } } @@ -93,9 +97,7 @@ public class RankSetupValidator extends Validator { } private void deleteTempDir(File dir) { - if (!IOUtils.recursiveDeleteDir(dir)) { - throw new RuntimeException("Failed deleting " + dir); - } + IOUtils.recursiveDeleteDir(dir); } private void writeConfigs(String dir, AbstractConfigProducer producer) throws IOException { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index e60aabd24e8..659a07cfd5c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -60,7 +60,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl return s.toString(); } - private int getGCInterval(ModelElement documentNode) throws ParseException { + private int getGCInterval(ModelElement documentNode) { int gcInterval = 3600; if (documentNode != null) { gcInterval = documentNode.getIntegerAttribute("garbage-collection-interval", gcInterval); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java new file mode 100644 index 00000000000..bc812e12f43 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java @@ -0,0 +1,63 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class ImplicitIndexingClusterTest { + @Test + public void existing_jdisc_is_used_as_indexing_cluster_when_multitenant() { + final String servicesXml = "<services version=\"1.0\">\n" + // + " <jdisc version=\"1.0\" id=\"jdisc\">\n" + // + " <search />\n" + // + " <nodes count=\"1\" />\n" + // + ACCESS_CONTROL_XML + // + " </jdisc>\n" + // + " <content id=\"music\" version=\"1.0\">\n" + // + " <redundancy>1</redundancy>\n" + // + " <documents>\n" + // + " <document type=\"music\" mode=\"index\" />\n" + // + " </documents>\n" + // + " <nodes count=\"1\" />\n" + // + " </content>\n" + // + "</services>\n"; + + + VespaModel vespaModel = buildMultiTenantVespaModel(servicesXml); + ContainerCluster jdisc = vespaModel.getContainerClusters().get("jdisc"); + assertNotNull("Docproc not added to jdisc", jdisc.getDocproc()); + assertNotNull("Indexing chain not added to jdisc", jdisc.getDocprocChains().allChains().getComponent("indexing")); + } + + private final String ACCESS_CONTROL_XML = "<http>\n" +// + " <filtering>\n" +// + " <access-control domain=\"foo\" />\n" +// + " </filtering>\n" +// + " <server id=\"bar\" port=\"4080\" />\n" +// + "</http>\n"; + + private static VespaModel buildMultiTenantVespaModel(String servicesXml) { + DeployProperties properties = new DeployProperties.Builder().multitenant(true).hostedVespa(true).build(); + DeployState.Builder deployStateBuilder = new DeployState.Builder() + .properties(properties) + .modelHostProvisioner(new InMemoryProvisioner(true, "host1.yahoo.com", "host2.yahoo.com", "host3.yahoo.com")); + + return new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withServices("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + servicesXml) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.java new file mode 100644 index 00000000000..bc6ff9e9be4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.java @@ -0,0 +1,180 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel.TargetSpec; +import com.yahoo.vespa.model.ConfigProducer; +import com.yahoo.vespa.model.container.search.searchchain.Source.GroupOption; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; +import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class FederationSearcherTest { + + private static class FederationFixture { + FederationSearcher federationSearchWithDefaultSources = newFederationSearcher(true, emptyList()); + private ComponentRegistry<SearchChain> searchChainRegistry = new ComponentRegistry<>(); + private SourceGroupRegistry sourceGroupRegistry = new SourceGroupRegistry(); + + void initializeFederationSearcher(FederationSearcher searcher) { + searcher.initialize(searchChainRegistry, sourceGroupRegistry); + } + + void registerProviderWithSources(Provider provider) { + List<GenericTarget> sources = new ArrayList<>(); + sources.add(provider); + sources.addAll(provider.getSources()); + for (GenericTarget gt : sources) { + searchChainRegistry.register(gt.getId(), gt); + } + sourceGroupRegistry.addSources(provider); + } + } + + private static class ProvidersWithSourceFixture extends FederationFixture { + Provider provider1 = createProvider(ComponentId.fromString("provider1")); + Provider provider2 = createProvider(ComponentId.fromString("provider2")); + + private ProvidersWithSourceFixture() { + super(); + provider1.addSource(createSource(ComponentId.fromString("source"), GroupOption.leader)); + provider2.addSource(createSource(ComponentId.fromString("source"), GroupOption.participant)); + + registerProviderWithSources(provider1); + registerProviderWithSources(provider2); + initializeFederationSearcher(federationSearchWithDefaultSources); + } + } + + @Test + public void default_providers_are_inherited_when_inheritDefaultSources_is_true() throws Exception { + FederationFixture f = new FederationFixture(); + + final String providerId = "providerId"; + + f.registerProviderWithSources(createProvider(ComponentId.fromString(providerId))); + f.initializeFederationSearcher(f.federationSearchWithDefaultSources); + + FederationConfig federationConfig = getConfig(f.federationSearchWithDefaultSources); + FederationConfig.Target target = federationConfig.target(0); + + assertSame(providerId, target.id()); // by identity + assertTrue("Not used by default", target.searchChain(0).useByDefault()); + } + + @Test + public void source_groups_are_inherited_when_inheritDefaultSources_is_true() throws Exception { + FederationFixture f = new ProvidersWithSourceFixture(); + + FederationConfig federationConfig = getConfig(f.federationSearchWithDefaultSources); + Assert.assertEquals(1, federationConfig.target().size()); + + FederationConfig.Target target = federationConfig.target(0); + assertEquals(target.id(), "source"); + assertTrue("Not used by default", target.useByDefault()); + assertEquals(2, target.searchChain().size()); + assertThat(target.searchChain().stream().map(FederationConfig.Target.SearchChain::providerId).collect(toList()), + contains("provider1", "provider2")); + } + + @Test + public void source_groups_are_not_inherited_when_inheritDefaultSources_is_false() throws Exception { + FederationFixture f = new ProvidersWithSourceFixture(); + + FederationSearcher federationSearcherWithoutDefaultSources = newFederationSearcher(false, emptyList()); + f.initializeFederationSearcher(federationSearcherWithoutDefaultSources); + + FederationConfig federationConfig = getConfig(federationSearcherWithoutDefaultSources); + assertEquals(0, federationConfig.target().size()); + } + + @Test + public void leaders_must_be_the_first_search_chain_in_a_target() throws Exception { + FederationFixture f = new ProvidersWithSourceFixture(); + + FederationConfig federationConfig = getConfig(f.federationSearchWithDefaultSources); + List<FederationConfig.Target.SearchChain> searchChain = federationConfig.target(0).searchChain(); + + assertEquals("provider1", searchChain.get(0).providerId()); + assertEquals("provider2", searchChain.get(1).providerId()); + } + + @Test + public void manually_specified_targets_overrides_inherited_targets() throws Exception { + FederationFixture f = new FederationFixture(); + + f.registerProviderWithSources(createProvider(ComponentId.fromString("provider1"))); + FederationSearcher federation = newFederationSearcher(true, + singletonList(new TargetSpec(ComponentSpecification.fromString("provider1"), + new FederationOptions().setTimeoutInMilliseconds(12345)))); + f.initializeFederationSearcher(federation); + + FederationConfig federationConfig = getConfig(federation); + assertEquals(1, federationConfig.target().size()); + + FederationConfig.Target target = federationConfig.target(0); + assertEquals(1, target.searchChain().size()); + + FederationConfig.Target.SearchChain searchChain = target.searchChain(0); + assertEquals(12345, searchChain.timeoutMillis()); + } + + private static FederationSearcher newFederationSearcher(boolean inheritDefaultSources, List<TargetSpec> targets) { + return new FederationSearcher(new FederationSearcherModel(ComponentSpecification.fromString("federation"), + Dependencies.emptyDependencies(), targets, inheritDefaultSources), Optional.empty()); + } + + private static ChainSpecification searchChainSpecification(ComponentId id) { + return new ChainSpecification(id, new ChainSpecification.Inheritance(null, null), emptyList(), emptySet()); + } + + private static Provider createProvider(ComponentId id) { + return new Provider(searchChainSpecification(id), new FederationOptions()); + } + + private static Source createSource(ComponentId id, GroupOption groupOption) { + return new Source(searchChainSpecification(id), new FederationOptions(), groupOption); + } + + private static FederationConfig getConfig(ConfigProducer configProducer) throws Exception { + Optional<Class<?>> builderClassOpt = Arrays.stream(FederationConfig.class.getDeclaredClasses()) + .filter(c -> c.getSimpleName().equals("Builder")).findFirst(); + if (builderClassOpt.isPresent() == false) { + throw new RuntimeException("No Builder class in ConfigInstance."); + } + Class<?> builderClass = builderClassOpt.get(); + + Object builder = builderClass.getDeclaredConstructor().newInstance(); + Method getConfigMethod = configProducer.getClass().getMethod("getConfig", builderClass); + + getConfigMethod.invoke(configProducer, builder); + + return FederationConfig.class.getConstructor(builderClass).newInstance(builder); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java new file mode 100644 index 00000000000..c58f3308ced --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -0,0 +1,59 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.search.grouping.GroupingValidator; +import org.apache.commons.io.IOUtils; +import org.junit.Test; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.InputStream; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + * @author ollivir + */ + +public class BundleInstantiationSpecificationBuilderTest { + + @Test + public void bundle_is_not_replaced_for_user_defined_class() throws IOException, SAXException { + final String userDefinedClass = "my own class that will also be set as bundle"; + verifyExpectedBundle(userDefinedClass, null, userDefinedClass); + } + + @Test + public void bundle_is_replaced_for_internal_class() throws IOException, SAXException { + String internalClass = GroupingValidator.class.getName(); + verifyExpectedBundle(internalClass, null, BundleMapper.searchAndDocprocBundle); + } + + @Test + public void bundle_is_not_replaced_for_internal_class_with_explicitly_set_bundle() + throws IOException, SAXException { + String internalClass = GroupingValidator.class.getName(); + String explicitBundle = "my-own-implementation"; + verifyExpectedBundle(internalClass, explicitBundle, explicitBundle); + } + + private static void verifyExpectedBundle(String className, String explicitBundle, String expectedBundle) + throws IOException, SAXException { + String xml = "<component id=\"_\" class=\"" + className + "\""; + if (explicitBundle != null) { + xml += " bundle=\"" + explicitBundle + "\""; + } + xml += " />"; + InputStream xmlStream = IOUtils.toInputStream(xml); + Element component = XmlHelper.getDocumentBuilder().parse(xmlStream).getDocumentElement(); + + BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component, false); + assertThat(spec.bundle, is(ComponentSpecification.fromString(expectedBundle))); + } +} diff --git a/config-model/src/test/scala/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.scala b/config-model/src/test/scala/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.scala deleted file mode 100644 index 4ebe14c1e85..00000000000 --- a/config-model/src/test/scala/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.scala +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.search - -import com.yahoo.config.model.provision.InMemoryProvisioner -import com.yahoo.config.model.test.MockApplicationPackage -import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg -import org.junit.Test - -import org.junit.Assert.assertNotNull -import scala.xml.{XML, Elem} -import java.io.StringWriter -import com.yahoo.config.model.deploy.{DeployProperties, DeployState} - -/** - * @author tonytv - */ -class ImplicitIndexingClusterTest { - @Test - def existing_jdisc_is_used_as_indexing_cluster_when_multitenant() { - val servicesXml = - <services version="1.0"> - <jdisc version="1.0" id="jdisc"> - <search /> - <nodes count="1" /> - {accessControlXml} - </jdisc> - <content id="music" version="1.0"> - <redundancy>1</redundancy> - <documents> - <document type="music" mode="index" /> - </documents> - <nodes count="1" /> - </content> - </services> - - - val vespaModel = buildMultiTenantVespaModel(servicesXml) - val jdisc = vespaModel.getContainerClusters.get("jdisc") - assertNotNull("Docproc not added to jdisc", jdisc.getDocproc) - assertNotNull("Indexing chain not added to jdisc", jdisc.getDocprocChains.allChains().getComponent("indexing")) - } - - private val accessControlXml = - <http> - <filtering> - <access-control domain="foo" /> - </filtering> - <server id="bar" port="4080" /> - </http> - - - def buildMultiTenantVespaModel(servicesXml: Elem) = { - val properties = new DeployProperties.Builder().multitenant(true).hostedVespa(true).build() - val deployStateBuilder = new DeployState.Builder() - .properties(properties) - .modelHostProvisioner(new InMemoryProvisioner(true, "host1.yahoo.com", "host2.yahoo.com", "host3.yahoo.com")) - - val writer = new StringWriter - XML.write(writer, servicesXml, "UTF-8", xmlDecl = true, doctype = null) - writer.close() - - new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withServices(writer.toString) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(deployStateBuilder) - } -} diff --git a/config-model/src/test/scala/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.scala b/config-model/src/test/scala/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.scala deleted file mode 100644 index a1eeb297367..00000000000 --- a/config-model/src/test/scala/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.scala +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.search.searchchain - -import java.util.Collections.{emptyList, emptySet} -import java.util.Optional - -import com.yahoo.component.chain.dependencies.Dependencies -import com.yahoo.component.chain.model.ChainSpecification -import com.yahoo.component.provider.ComponentRegistry -import com.yahoo.component.{ComponentId, ComponentSpecification} -import com.yahoo.config.ConfigInstance -import com.yahoo.search.federation.FederationConfig -import com.yahoo.search.searchchain.model.federation.FederationSearcherModel.TargetSpec -import com.yahoo.search.searchchain.model.federation.{FederationOptions, FederationSearcherModel} -import com.yahoo.vespa.model.ConfigProducer -import com.yahoo.vespa.model.container.search.searchchain.FederationSearcherTest._ -import com.yahoo.vespa.model.container.search.searchchain.Source.GroupOption -import org.junit.runner.RunWith -import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner - -import scala.collection.JavaConverters._ -import scala.collection.breakOut -import scala.language.implicitConversions -import scala.reflect.ClassTag - -/** - * @author tonytv - */ -@RunWith(classOf[JUnitRunner]) -class FederationSearcherTest extends FunSuite{ - - class FederationFixture { - val federationSearchWithDefaultSources = newFederationSearcher(inheritDefaultSources = true) - val searchChainRegistry = new ComponentRegistry[SearchChain] - val sourceGroupRegistry = new SourceGroupRegistry - - def initializeFederationSearcher(searcher: FederationSearcher = federationSearchWithDefaultSources) { - searcher.initialize(searchChainRegistry, sourceGroupRegistry) - } - - def registerProviderWithSources(provider: Provider) = { - provider :: provider.getSources.asScala.toList foreach { chain => searchChainRegistry.register(chain.getId, chain) } - sourceGroupRegistry.addSources(provider) - } - } - - class ProvidersWithSourceFixture extends FederationFixture { - val provider1 = createProvider("provider1") - val provider2 = createProvider("provider2") - - provider1.addSource(createSource("source", GroupOption.leader)) - provider2.addSource(createSource("source", GroupOption.participant)) - - registerProviderWithSources(provider1) - registerProviderWithSources(provider2) - initializeFederationSearcher() - } - - test("default providers are inherited when inheritDefaultSources=true") { - val f = new FederationFixture - import f._ - - val providerId = "providerId" - - registerProviderWithSources(createProvider(providerId)) - initializeFederationSearcher() - - val federationConfig = getConfig[FederationConfig](federationSearchWithDefaultSources) - val target = federationConfig.target(0) - - assert( providerId === target.id() ) - assert( target.searchChain(0).useByDefault(), "Not used by default" ) - } - - def toMapByKey[KEY, VALUE](collection: java.util.Collection[VALUE])(f: VALUE => KEY): Map[KEY, VALUE] = - collection.asScala.map(e => (f(e), e))(breakOut) - - test("source groups are inherited when inheritDefaultSources=true") { - val f = new ProvidersWithSourceFixture - import f._ - - val federationConfig = getConfig[FederationConfig](federationSearchWithDefaultSources) - assert(federationConfig.target().size == 1) - - val target = federationConfig.target(0) - assert(target.id() == "source") - assert(target.useByDefault(), "Not used by default") - - //val chainsByProviderId = toMapByKey(target.searchChain())(_.providerId()) - - assert(Set("provider1", "provider2") === target.searchChain().asScala.map(_.providerId()).toSet) - } - - test("source groups are not inherited when inheritDefaultSources=false") { - val f = new ProvidersWithSourceFixture - import f._ - - val federationSearcherWithoutDefaultSources = newFederationSearcher(inheritDefaultSources = false) - initializeFederationSearcher(federationSearcherWithoutDefaultSources) - - val federationConfig = getConfig[FederationConfig](federationSearcherWithoutDefaultSources) - assert(federationConfig.target().size == 0) - } - - test("leaders must be the first search chain in a target") { - val f = new ProvidersWithSourceFixture - import f._ - - val federationConfig = getConfig[FederationConfig](federationSearchWithDefaultSources) - val searchChain = federationConfig.target(0).searchChain - - assert(searchChain.get(0).providerId() === "provider1") - assert(searchChain.get(1).providerId() === "provider2") - - } - - test("manually specified targets overrides inherited targets") { - val f = new FederationFixture - import f._ - - registerProviderWithSources(createProvider("provider1")) - val federation = newFederationSearcher(inheritDefaultSources = true, - targets = List(new TargetSpec("provider1", new FederationOptions().setTimeoutInMilliseconds(12345))).asJava) - - initializeFederationSearcher(federation) - - val federationConfig = getConfig[FederationConfig](federation) - - assert(federationConfig.target().size === 1) - val target = federationConfig.target(0) - - assert(target.searchChain().size === 1) - val searchChain = target.searchChain(0) - - assert(searchChain.timeoutMillis() === 12345) - } - - - def newFederationSearcher(inheritDefaultSources: Boolean, - targets: java.util.List[TargetSpec] = emptyList()): FederationSearcher = { - new FederationSearcher( - new FederationSearcherModel("federation", - Dependencies.emptyDependencies(), - targets, - inheritDefaultSources), - Optional.empty()) - } -} - -object FederationSearcherTest { - implicit def toComponentId(name: String): ComponentId = ComponentId.fromString(name) - implicit def toComponentSpecification(name: String): ComponentSpecification = ComponentSpecification.fromString(name) - - def newBuilder[T <: ConfigInstance.Builder](implicit c: ClassTag[T]): T = { - c.runtimeClass.getDeclaredConstructor().newInstance().asInstanceOf[T] - } - - def searchChainSpecification(id: ComponentId) = - new ChainSpecification(id, new ChainSpecification.Inheritance(null, null), emptyList(), emptySet()) - - def createProvider(id: ComponentId) = - new Provider(searchChainSpecification(id), new FederationOptions()) - - def createSource(id: ComponentId, groupOption: GroupOption) = - new Source(searchChainSpecification(id), new FederationOptions(), groupOption) - - - //TODO: TVT: move - def getConfig[T <: ConfigInstance : ClassTag](configProducer: ConfigProducer): T = { - val configClass = implicitly[ClassTag[T]].runtimeClass - val builderClass = configClass.getDeclaredClasses.collectFirst {case c if c.getSimpleName == "Builder" => c } getOrElse { - sys.error("No Builder class in ConfigInstance.") - } - - val builder = builderClass.getDeclaredConstructor().newInstance().asInstanceOf[AnyRef] - val getConfigMethod = configProducer.getClass.getMethod("getConfig", builderClass) - - getConfigMethod.invoke(configProducer, builder) - - configClass.getConstructor(builderClass).newInstance(builder).asInstanceOf[T] - } -} diff --git a/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.scala b/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.scala deleted file mode 100644 index 1f849f91c2e..00000000000 --- a/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.scala +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.xml - -import com.yahoo.component.ComponentSpecification -import com.yahoo.search.grouping.GroupingValidator - -import scala.language.implicitConversions -import BundleInstantiationSpecificationBuilderTest._ -import com.yahoo.config.model.builder.xml.test.DomBuilderTest -import org.hamcrest.CoreMatchers._ -import org.junit.Assert._ -import org.junit.Test -import org.w3c.dom.Element - -import scala.xml.Elem - -/** - * @author gjoranv - * @since 5.45 - */ - -class BundleInstantiationSpecificationBuilderTest { - - @Test - def bundle_is_not_replaced_for_user_defined_class() { - val userDefinedClass = "my own class that will also be set as bundle" - verifyExpectedBundle(userDefinedClass, - expectedBundle = userDefinedClass) - } - - @Test - def bundle_is_replaced_for_internal_class() = { - val internalClass = classOf[GroupingValidator].getName - verifyExpectedBundle(internalClass, - expectedBundle = BundleMapper.searchAndDocprocBundle) - } - - @Test - def bundle_is_not_replaced_for_internal_class_with_explicitly_set_bundle() = { - val internalClass = classOf[GroupingValidator].getName - val explicitBundle = "my-own-implementation" - verifyExpectedBundle(internalClass, - explicitBundle = Some(explicitBundle), - expectedBundle = explicitBundle) - } -} - -object BundleInstantiationSpecificationBuilderTest { - - def verifyExpectedBundle(className: String, - explicitBundle: Option[String] = None, - expectedBundle:String) = { - val xml = <component id="_" class={className} bundle={explicitBundle.orNull} /> - - val spec = BundleInstantiationSpecificationBuilder.build(xml, false) - assertThat(spec.bundle, is(ComponentSpecification.fromString(expectedBundle))) - } - - implicit def toDomElement(elem: Elem): Element = { - DomBuilderTest.parse(elem.toString()) - } -} diff --git a/config/src/apps/vespa-get-config/getconfig.cpp b/config/src/apps/vespa-get-config/getconfig.cpp index 342b2b497bb..24e4623372e 100644 --- a/config/src/apps/vespa-get-config/getconfig.cpp +++ b/config/src/apps/vespa-get-config/getconfig.cpp @@ -164,7 +164,7 @@ GetConfig::Main() break; case 'h': retval = 0; - //@fallthrough@ + [[fallthrough]]; case '?': default: usage(); diff --git a/config/src/apps/vespa-ping-configproxy/pingproxy.cpp b/config/src/apps/vespa-ping-configproxy/pingproxy.cpp index 86b5a4a13ba..a88d7deb79a 100644 --- a/config/src/apps/vespa-ping-configproxy/pingproxy.cpp +++ b/config/src/apps/vespa-ping-configproxy/pingproxy.cpp @@ -99,7 +99,7 @@ PingProxy::Main() case '?': default: retval = 1; - // fallthrough + [[fallthrough]]; case 'h': usage(); return retval; diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 7a1ddeb8f66..5e81526dc53 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -52,5 +52,6 @@ ztsUrl string default="" nodeAdminInContainer bool default=true # Maintainers +maintainerIntervalMinutes int default=30 # TODO: Default set to a high value (1 year) => maintainer will not run, change when maintainer verified out in prod tenantsMaintainerIntervalMinutes int default=525600 diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 549af4d1f64..cef59809248 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -153,7 +153,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams, boolean ignoreLockFailure, boolean ignoreSessionStaleFailure, Instant now) { - return deploy(decompressApplication(in), prepareParams, ignoreLockFailure, ignoreSessionStaleFailure, now); + File tempDir = Files.createTempDir(); + PrepareResult prepareResult; + try { + prepareResult = deploy(decompressApplication(in, tempDir), prepareParams, ignoreLockFailure, ignoreSessionStaleFailure, now); + } finally { + cleanupTempDirectory(tempDir); + } + return prepareResult; } public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams) { @@ -349,7 +356,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, InputStream in, String contentType) { - return createSession(applicationId, timeoutBudget, decompressApplication(in, contentType)); + File tempDir = Files.createTempDir(); + long sessionId; + try { + sessionId = createSession(applicationId, timeoutBudget, decompressApplication(in, contentType, tempDir)); + } finally { + cleanupTempDirectory(tempDir); + } + return sessionId; } public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File applicationDirectory) { @@ -440,21 +454,19 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return currentActiveApplicationSet; } - private File decompressApplication(InputStream in, String contentType) { + private File decompressApplication(InputStream in, String contentType, File tempDir) { try (CompressedApplicationInputStream application = CompressedApplicationInputStream.createFromCompressedStream(in, contentType)) { - return decompressApplication(application); + return decompressApplication(application, tempDir); } catch (IOException e) { throw new IllegalArgumentException("Unable to decompress data in body", e); } } - private File decompressApplication(CompressedApplicationInputStream in) { - File tempDir = Files.createTempDir(); + private File decompressApplication(CompressedApplicationInputStream in, File tempDir) { try { return in.decompress(tempDir); } catch (IOException e) { - cleanupTempDirectory(tempDir, logger); throw new IllegalArgumentException("Unable to decompress stream", e); } } @@ -464,7 +476,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return applicationRepo.listApplications(); } - private static void cleanupTempDirectory(File tempDir, DeployLogger logger) { + private void cleanupTempDirectory(File tempDir) { logger.log(LogLevel.DEBUG, "Deleting tmp dir '" + tempDir + "'"); if (!IOUtils.recursiveDeleteDir(tempDir)) { logger.log(LogLevel.WARNING, "Not able to delete tmp dir '" + tempDir + "'"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index c8b3bc824a8..2c46f2968ce 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.maintenance; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.curator.Curator; @@ -11,19 +12,41 @@ import java.time.Duration; public class ConfigServerMaintenance extends AbstractComponent { private final TenantsMaintainer tenantsMaintainer; + private final ZooKeeperDataMaintainer zooKeeperDataMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ConfigServerMaintenance(ConfigserverConfig configserverConfig, ApplicationRepository applicationRepository, Curator curator) { - tenantsMaintainer = new TenantsMaintainer(applicationRepository, - curator, - Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes())); + DefaultTimes defaults = new DefaultTimes(configserverConfig); + tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval); + zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.zookeeperDataMaintainerInterval); } @Override public void deconstruct() { tenantsMaintainer.deconstruct(); + zooKeeperDataMaintainer.deconstruct(); + } + + /* + * Default values from config. If one of the values needs to be changed, add the value to + * configserver-config.xml in the config server application directory and restart the config server + */ + private static class DefaultTimes { + + private final Duration defaultInterval; + private final Duration tenantsMaintainerInterval; + private final Duration zookeeperDataMaintainerInterval; + + DefaultTimes(ConfigserverConfig configserverConfig) { + boolean isCd = configserverConfig.system().equals(SystemName.cd.name()); + + this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes()); + // TODO: Want job control or feature flag to control when to run this, for now use a very long interval unless in CD + this.tenantsMaintainerInterval = isCd ? defaultInterval : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes()); + this.zookeeperDataMaintainerInterval = defaultInterval; + } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java index ce0811184a3..ae000385dfd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java @@ -20,7 +20,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { protected static final Logger log = Logger.getLogger(Maintainer.class.getName()); private static final Path root = Path.fromString("/configserver/v1/"); - private static final com.yahoo.path.Path lockRoot = root.append("locks"); + private static final Path lockRoot = root.append("locks"); private final Duration maintenanceInterval; private final ScheduledExecutorService service; @@ -36,13 +36,10 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { } @Override - @SuppressWarnings("try") + @SuppressWarnings({"try", "unused"}) public void run() { - try { - Path path = lockRoot.append(name()); - try (Lock lock = new Lock(path.toString(), curator)) { - maintain(); - } + try (Lock lock = lock(lockRoot.append(name()))) { + maintain(); } catch (UncheckedTimeoutException e) { // another config server instance is running this job at the moment; ok } catch (Throwable t) { @@ -50,6 +47,12 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { } } + private Lock lock(Path path) { + Lock lock = new Lock(path.getAbsolute(), curator); + lock.acquire(Duration.ofSeconds(1)); + return lock; + } + @Override public void deconstruct() { this.service.shutdown(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java index e06bf530486..36306dbdde8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java @@ -8,7 +8,7 @@ import java.time.Duration; public class TenantsMaintainer extends Maintainer { - public TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { + TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { super(applicationRepository, curator, interval); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java new file mode 100644 index 00000000000..852768b6937 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java @@ -0,0 +1,23 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.curator.Curator; + +import java.time.Duration; + +/** + * Removes unused zookeeper data (for now only data used by old file distribution code is removed) + */ +public class ZooKeeperDataMaintainer extends Maintainer { + + ZooKeeperDataMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { + super(applicationRepository, curator, interval); + } + + @Override + protected void maintain() { + curator.delete(Path.fromString("/vespa/filedistribution")); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 3010f1383da..551987f4dca 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.modelfactory; +import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.api.HostProvisioner; @@ -108,7 +109,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { catch (RuntimeException e) { boolean isOldestMajor = i == majorVersions.size() - 1; if (isOldestMajor) { - if (e instanceof NullPointerException || e instanceof NoSuchElementException) { + if (e instanceof NullPointerException || e instanceof NoSuchElementException | e instanceof UncheckedTimeoutException) { log.log(LogLevel.WARNING, "Unexpected error when building model ", e); throw new InternalServerException(applicationId + ": Error loading model", e); } else { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java index ed1eba58dfb..117930c6d7d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java @@ -23,18 +23,19 @@ import java.util.logging.Logger; public class SessionStateWatcher implements NodeCacheListener { private static final Logger log = Logger.getLogger(SessionStateWatcher.class.getName()); + // One thread pool for all instances of this class + private static final Executor executor = Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(SessionStateWatcher.class.getName())); + private final Curator.FileCache fileCache; private final ReloadHandler reloadHandler; private final RemoteSession session; private final MetricUpdater metrics; - private final Executor executor; + public SessionStateWatcher(Curator.FileCache fileCache, ReloadHandler reloadHandler, RemoteSession session, MetricUpdater metrics) { - executor = Executors.newSingleThreadExecutor( - ThreadFactoryFactory.getThreadFactory(SessionStateWatcher.class.getName() + "-" + session)); this.fileCache = fileCache; this.reloadHandler = reloadHandler; this.session = session; diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index b984ce60702..9e01f594484 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -53,8 +53,8 @@ <preprocess:include file='hosted-vespa/routing-status.xml' required='false' /> <preprocess:include file='hosted-vespa/scoreboard.xml' required='false' /> <preprocess:include file='controller/container.xml' required='false' /> - <component id="com.yahoo.vespa.service.monitor.internal.SlobrokMonitorManagerImpl" bundle="service-monitor" /> - <component id="com.yahoo.vespa.service.monitor.internal.HealthMonitorManager" bundle="service-monitor" /> + <component id="com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl" bundle="service-monitor" /> + <component id="com.yahoo.vespa.service.monitor.internal.health.HealthMonitorManager" bundle="service-monitor" /> <component id="com.yahoo.vespa.service.monitor.internal.ServiceMonitorImpl" bundle="service-monitor" /> <component id="com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService" bundle="orchestrator" /> <component id="com.yahoo.vespa.orchestrator.status.ZookeeperStatusService" bundle="orchestrator" /> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java new file mode 100644 index 00000000000..b92feffbb55 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; + +import java.time.Clock; + +class MaintainerTester { + + private final Curator curator; + private final TenantRepository tenantRepository; + private final ApplicationRepository applicationRepository; + + MaintainerTester() { + curator = new MockCurator(); + GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); + tenantRepository = new TenantRepository(componentRegistry, false); + applicationRepository = new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), Clock.systemUTC()); + } + + Curator curator() { return curator; } + TenantRepository tenantRepository() { return tenantRepository; } + + ApplicationRepository applicationRepository() { return applicationRepository;} + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java new file mode 100644 index 00000000000..80d9f808bfc --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java @@ -0,0 +1,45 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import org.junit.Test; + +import java.io.File; +import java.time.Duration; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class TenantsMaintainerTest { + + @Test + public void deleteTenantWithNoApplications() { + MaintainerTester tester = new MaintainerTester(); + TenantRepository tenantRepository = tester.tenantRepository(); + ApplicationRepository applicationRepository = tester.applicationRepository(); + + TenantName shouldBeDeleted = TenantName.from("to-be-deleted"); + TenantName shouldNotBeDeleted = TenantName.from("should-not-be-deleted"); + + tenantRepository.addTenant(shouldBeDeleted); + tenantRepository.addTenant(shouldNotBeDeleted); + applicationRepository.deploy(new File("src/test/apps/app"), + new PrepareParams.Builder() + .applicationId(ApplicationId.from(shouldNotBeDeleted, ApplicationName.from("foo"), InstanceName.defaultName())) + .build()); + assertNotNull(tenantRepository.getTenant(shouldBeDeleted)); + assertNotNull(tenantRepository.getTenant(shouldNotBeDeleted)); + + new TenantsMaintainer(applicationRepository, tester.curator(), Duration.ofDays(1)).run(); + + // One tenant should now have been deleted + assertNull(tenantRepository.getTenant(shouldBeDeleted)); + assertNotNull(tenantRepository.getTenant(shouldNotBeDeleted)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java new file mode 100644 index 00000000000..1c886adde73 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java @@ -0,0 +1,30 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.Curator; +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ZooKeeperDataMaintainerTest { + + @Test + public void deleteOldData() { + MaintainerTester tester = new MaintainerTester(); + Curator curator = tester.curator(); + + curator.create(Path.fromString("/foo")); + curator.create(Path.fromString("/vespa/bar")); + curator.create(Path.fromString("/vespa/filedistribution")); + + new ZooKeeperDataMaintainer(tester.applicationRepository(), curator, Duration.ofDays(1)).run(); + + assertTrue(curator.exists(Path.fromString("/foo"))); + assertTrue(curator.exists(Path.fromString("/vespa"))); + assertFalse(curator.exists(Path.fromString("/vespa/filedistribution"))); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 42c2d8db968..9a4b0b05186 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -128,32 +128,6 @@ public class TenantRepositoryTest extends TestWithCurator { } @Test - public void testTenantsChanged() { - tenantRepository.close(); // Close the repo created in setup() - TenantRepository tenantRepository = new TenantRepository(globalComponentRegistry); - tenantRepository.addTenant(tenant2); - tenantRepository.createTenants(); - Set<TenantName> allTenants = tenantRepository.getAllTenantNames(); - assertTrue(allTenants.contains(tenant2)); - tenantRepository.deleteTenant(tenant1); - tenantRepository.deleteTenant(tenant2); - tenantRepository.createTenants(); - allTenants = tenantRepository.getAllTenantNames(); - assertFalse(allTenants.contains(tenant1)); - assertFalse(allTenants.contains(tenant2)); - TenantName foo = TenantName.from("foo"); - TenantName bar = TenantName.from("bar"); - tenantRepository.addTenant(tenant2); - tenantRepository.addTenant(foo); - tenantRepository.addTenant(bar); - tenantRepository.createTenants(); - allTenants = tenantRepository.getAllTenantNames(); - assertTrue(allTenants.contains(tenant2)); - assertTrue(allTenants.contains(foo)); - assertTrue(allTenants.contains(bar)); - } - - @Test public void testTenantWatching() throws Exception { TenantName newTenant = TenantName.from("newTenant"); List<TenantName> expectedTenants = Arrays.asList(TenantName.defaultName(), newTenant); diff --git a/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java b/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java index e4ec09e3948..31bceca9337 100644 --- a/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java +++ b/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.logging.Logger; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; @@ -37,6 +38,7 @@ import static java.util.stream.Collectors.toSet; * @author bjorncs */ public class FilterChainRepository extends AbstractComponent { + private static final Logger log = Logger.getLogger(FilterChainRepository.class.getName()); private final ComponentRegistry<Object> filterAndChains; @@ -77,12 +79,23 @@ public class FilterChainRepository extends AbstractComponent { ChainRegistry<FilterWrapper> chainRegistry = new ChainRegistry<>(); ChainsModel chainsModel = ChainsModelBuilder.buildFromConfig(chainsConfig); ChainsConfigurer.prepareChainRegistry(chainRegistry, chainsModel, allFiltersWrapped(filters)); + removeEmptyChains(chainRegistry); chainRegistry.freeze(); return chainRegistry; } + private static void removeEmptyChains(ChainRegistry<FilterWrapper> chainRegistry) { + chainRegistry.allComponents().stream() + .filter(chain -> chain.components().isEmpty()) + .map(Chain::getId) + .peek(id -> log.warning("Removing empty filter chain: " + id)) + .forEach(chainRegistry::unregister); + } + @SuppressWarnings("unchecked") private static Object toJDiscChain(Chain<FilterWrapper> chain) { + if (chain.components().isEmpty()) + throw new IllegalArgumentException("Empty filter chain: " + chain.getId()); checkFilterTypesCompatible(chain); List<?> jdiscFilters = chain.components().stream() .map(filterWrapper -> filterWrapper.filter) @@ -98,7 +111,6 @@ public class FilterChainRepository extends AbstractComponent { } private static List<?> wrapSecurityFilters(List<?> filters) { - if (filters.isEmpty()) return emptyList(); List<Object> aggregatedSecurityFilters = new ArrayList<>(); List<Object> wrappedFilters = new ArrayList<>(); for (Object filter : filters) { diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 1f94372fc14..b4af6800768 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -464,7 +464,7 @@ <guava.version>18.0</guava.version> <guice.version>3.0</guice.version> <jaxb.version>2.2.7</jaxb.version> - <jetty.version>9.4.9.v20180320</jetty.version> + <jetty.version>9.4.10.v20180503</jetty.version> <scala.version>2.11.12</scala.version> <!-- When updating this, the scala.major-version in parent must also be updated! --> <slf4j.version>1.7.5</slf4j.version> diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index b899f690ee1..bdf395c1f0b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -166,8 +166,8 @@ public class IndexFacts { } private Index getIndexFromDocumentTypes(String indexName, List<String> documentTypes) { - if (indexName==null || indexName.isEmpty()) - indexName="default"; + if (indexName == null || indexName.isEmpty()) + indexName = "default"; return getIndexByCanonicNameFromDocumentTypes(indexName, documentTypes); } @@ -191,6 +191,13 @@ public class IndexFacts { return Index.nullIndex; } + private Collection<Index> getIndexes(String documentType) { + if ( ! isInitialized()) return Collections.emptyList(); + SearchDefinition sd = searchDefinitions.get(documentType); + if (sd == null) return Collections.emptyList(); + return sd.indices().values(); + } + /** Calls resolveDocumentTypes(query.getModel().getSources(), query.getModel().getRestrict()) */ private Set<String> resolveDocumentTypes(Query query) { // Assumption: Search definition name equals document name. @@ -421,6 +428,11 @@ public class IndexFacts { return IndexFacts.this.getIndexFromDocumentTypes(indexName, Collections.singletonList(documentType)); } + /** Returns all the indexes of a given search definition */ + public Collection<Index> getIndexes(String documentType) { + return IndexFacts.this.getIndexes(documentType); + } + /** * Returns the canonical form of the index name (Which may be the same as * the input). diff --git a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java index 644cacfa322..47becde7b19 100644 --- a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java @@ -83,7 +83,7 @@ public class SearchDefinition { } /** Returns the indices of this as a map */ - public Map<String,Index> indices() { + public Map<String, Index> indices() { return indices; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 0b3ddf689d9..c517742f0e5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -98,16 +98,18 @@ public class FastHit extends Hit { if (uri != null) return uri; // TODO: Remove on Vespa 7, this should be one of the last vestiges of URL field magic - if (fields().containsKey("uri")) { - // trigger decoding - Object o = getField("uri"); - setId(o.toString()); + Object uriField = getField("uri"); + if (uriField != null) { + setId(uriField.toString()); return super.getId(); } // Fallback to index:[source]/[partid]/[id] if (indexUri != null) return indexUri; - indexUri = new URI("index:" + getSource() + "/" + getPartId() + "/" + asHexString(getGlobalId())); + StringBuilder sb = new StringBuilder(64); + sb.append("index:").append(getSource()).append('/').append(getPartId()).append('/'); + asHexString(sb, getGlobalId()); + indexUri = new URI(sb.toString()); return indexUri; } @@ -348,7 +350,10 @@ public class FastHit extends Hit { /** @deprecated do not use */ @Deprecated // TODO: Make private on Vespa 7 public static String asHexString(GlobalId gid) { - StringBuilder sb = new StringBuilder(); + return asHexString(new StringBuilder(), gid).toString(); + } + + private static StringBuilder asHexString(StringBuilder sb, GlobalId gid) { byte[] rawGid = gid.getRawId(); for (byte b : rawGid) { String hex = Integer.toHexString(0xFF & b); @@ -357,7 +362,7 @@ public class FastHit extends Hit { } sb.append(hex); } - return sb.toString(); + return sb; } /** A set view of all the field names in this hit. Add/addAll is not supported but remove is. */ diff --git a/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java b/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java index e38c30c25ac..9d852c8822d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java +++ b/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java @@ -4,8 +4,10 @@ package com.yahoo.prelude.logging; /** * Hollow compatibility class for com.yahoo.container.logging.AccessLogEntry. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated do not use */ +@Deprecated // TODO: Remove on Vespa 7 public class AccessLogEntry extends com.yahoo.container.logging.AccessLogEntry { public AccessLogEntry() { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index e8e0a07941e..0ba4133901a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -52,7 +52,7 @@ public abstract class Item implements Cloneable { WEIGHTEDSET(15), WEAK_AND(16), EXACT(17), - LEGACY_RISE_QUERY_NOT_USED_ANYMORE_BUT_DO_NOT_REUSE_FOR_A_WHILE(18), + SAME_ELEMENT(18), PURE_WEIGHTED_STRING(19), PURE_WEIGHTED_INTEGER(20), DOTPRODUCT(21), diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java index 44555d19cc5..c3689805dd7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java @@ -205,8 +205,7 @@ public class PhraseItem extends CompositeIndexedItem { } /** Phrase items uses a empty heading instead of "PHRASE " */ - protected void appendHeadingString(StringBuilder buffer) { - } + protected void appendHeadingString(StringBuilder buffer) { } protected void appendBodyString(StringBuilder buffer) { appendIndexString(buffer); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java new file mode 100644 index 00000000000..e1b5842529f --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java @@ -0,0 +1,73 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + + +import com.google.common.annotations.Beta; +import com.yahoo.protect.Validator; + +import java.util.Iterator; + +/** + * This represents a query where all terms are required to match in the same element id. + * The primary usecase is to allow efficient search in arrays and maps of struct. + * The common path is the field name containing the struct. + * @author baldersheim + */ +@Beta +public class SameElementItem extends CompositeIndexedItem { + + public SameElementItem(String commonPath) { + setIndexName(commonPath); + } + + @Override + public String getIndexedString() { + StringBuilder buf = new StringBuilder(); + + for (Iterator<Item> i = getItemIterator(); i.hasNext();) { + IndexedItem indexedItem = (IndexedItem) i.next(); + + buf.append(indexedItem.getIndexedString()); + if (i.hasNext()) { + buf.append(' '); + } + } + return buf.toString(); + } + + protected void appendHeadingString(StringBuilder buffer) { } + protected void appendBodyString(StringBuilder buffer) { + appendIndexString(buffer); + buffer.append('{'); + for (Iterator<Item> i = getItemIterator(); i.hasNext();) { + TermItem term = (TermItem) i.next(); + buffer.append(term.getIndexName()).append(':').append(term.getIndexedString()); + if (i.hasNext()) { + buffer.append(' '); + } + } + buffer.append('}'); + } + + @Override + public int getNumWords() { + return getItemCount(); + } + + @Override + protected void adding(Item item) { + Validator.ensureInstanceOf("Child item", item, TermItem.class); + TermItem asTerm = (TermItem) item; + Validator.ensureNonEmpty("Struct fieldname", asTerm.getIndexName()); + Validator.ensureNonEmpty("Query term", asTerm.getIndexedString()); + } + @Override + public ItemType getItemType() { + return ItemType.SAME_ELEMENT; + } + + @Override + public String getName() { + return getItemType().toString(); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/CollapsePhraseSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/CollapsePhraseSearcher.java index abf37c71b76..47e5651f64c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/CollapsePhraseSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/CollapsePhraseSearcher.java @@ -15,9 +15,11 @@ import com.yahoo.search.searchchain.Execution; /** * Make single item phrases in query into single word items. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class CollapsePhraseSearcher extends Searcher { + + @Override public Result search(Query query, Execution execution) { QueryTree tree = query.getModel().getQueryTree(); Item root = tree.getRoot(); @@ -35,7 +37,6 @@ public class CollapsePhraseSearcher extends Searcher { return execution.search(query); } - private Item simplifyPhrases(Item root) { if (root == null) { return root; @@ -64,4 +65,5 @@ public class CollapsePhraseSearcher extends Searcher { else return root; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java index 3d803b322ca..dd8f4eff666 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java @@ -22,14 +22,15 @@ import java.util.*; * Searcher to rewrite queries to achieve mixed recall between indices and * memory attributes. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated do not use */ @After({PhaseNames.RAW_QUERY, PHRASE_REPLACEMENT}) @Before(PhaseNames.TRANSFORMED_QUERY) @Provides(IndexCombinatorSearcher.MIXED_RECALL_REWRITE) -// TODO: This is not necessary on Vespa 6, we should probably remove it from the default chain but keep it -// around until Vespa 6 to avoid breaking those who refer to it. +@Deprecated // TODO: Remove on Vespa 7 (not necessary any more) public class IndexCombinatorSearcher extends Searcher { + public static final String MIXED_RECALL_REWRITE = "MixedRecallRewrite"; private static class ArrayComparator implements Comparator<Attribute[]> { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java index 72c38448936..7456f33d00f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java @@ -1,16 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.querytransform; - import java.util.List; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.search.Query; +import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.query.Sorting.FieldOrder; import com.yahoo.search.searchchain.Execution; - /** * Avoid doing relevance calculations if sorting only * on attributes. @@ -25,7 +25,7 @@ public class NoRankingSearcher extends Searcher { private static final String UNRANKED = "unranked"; @Override - public com.yahoo.search.Result search(com.yahoo.search.Query query, Execution execution) { + public Result search(Query query, Execution execution) { List<FieldOrder> s = (query.getRanking().getSorting() != null) ? query.getRanking().getSorting().fieldOrders() : null; if (s == null) { return execution.search(query); diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java index 7a548acbff7..ffb1b8a4965 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java @@ -5,6 +5,8 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.Query; +import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.searchchain.Execution; @@ -12,7 +14,7 @@ import com.yahoo.search.searchchain.Execution; import java.util.List; /** - * <p>Detects and removes certain phrases from the query.</p> + * Detects and removes certain phrases from the query. * * @author bratseth */ @@ -52,9 +54,9 @@ public class NonPhrasingSearcher extends Searcher { } @Override - public com.yahoo.search.Result search(com.yahoo.search.Query query, Execution execution) { - List<PhraseMatcher.Phrase> phrases=phraseMatcher.matchPhrases(query.getModel().getQueryTree().getRoot()); - if (phrases!=null && !query.properties().getBoolean(suggestonly, false)) { + public Result search(Query query, Execution execution) { + List<PhraseMatcher.Phrase> phrases = phraseMatcher.matchPhrases(query.getModel().getQueryTree().getRoot()); + if (phrases != null && !query.properties().getBoolean(suggestonly, false)) { remove(phrases); query.trace("Removing stop words",true,2); } @@ -64,9 +66,9 @@ public class NonPhrasingSearcher extends Searcher { private void remove(List<PhraseMatcher.Phrase> phrases) { // Removing the leaf replace phrases first to preserve // the start index of each replace phrase until removing - for (int i=phrases.size()-1; i>=0; i-- ) { - PhraseMatcher.Phrase phrase= phrases.get(i); - if (phrase.getLength()<phrase.getOwner().getItemCount()) // Don't removeField all + for (int i = phrases.size()-1; i >= 0; i-- ) { + PhraseMatcher.Phrase phrase = phrases.get(i); + if (phrase.getLength() < phrase.getOwner().getItemCount()) // Don't removeField all phrase.remove(); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java index 02c8ecda60c..fdd6ad47a98 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java @@ -11,6 +11,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexFacts.Session; import com.yahoo.prelude.query.*; import com.yahoo.prelude.query.WordAlternativesItem.Alternative; +import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; @@ -46,7 +47,7 @@ public class NormalizingSearcher extends Searcher { } @Override - public com.yahoo.search.Result search(com.yahoo.search.Query query, Execution execution) { + public Result search(Query query, Execution execution) { normalize(query, execution.context().getIndexFacts().newSession(query)); return execution.search(query); } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java index f4891489216..e8e4dc39fd5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java @@ -20,15 +20,15 @@ public class PhraseMatcher { private FSA phraseFSA = null; - private boolean matchPhraseItems=false; + private boolean matchPhraseItems = false; - private boolean matchSingleItems=false; + private boolean matchSingleItems = false; /** Whether this should ignore regular plural/singular form differences when matching */ - private boolean ignorePluralForm=false; + private boolean ignorePluralForm = false; /** False to matche the longest phrase, true to match <i>all</i> phrases */ - private boolean matchAll =false; + private boolean matchAll = false; /** For null subclass only */ private PhraseMatcher() { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java index d530ec6b45e..2f3f4dbd351 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java @@ -29,7 +29,7 @@ import java.util.List; @Provides(PhrasingSearcher.PHRASE_REPLACEMENT) public class PhrasingSearcher extends Searcher { - private static final CompoundName suggestonly=new CompoundName("suggestonly"); + private static final CompoundName suggestonly = new CompoundName("suggestonly"); public static final String PHRASE_REPLACEMENT = "PhraseReplacement"; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java index 4490d3c9b1e..69331a196a2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java @@ -38,7 +38,7 @@ public class RecallSearcher extends Searcher { private static final CompoundName recallName=new CompoundName("recall"); @Override - public com.yahoo.search.Result search(Query query, Execution execution) { + public Result search(Query query, Execution execution) { String recall = query.properties().getString(recallName); if (recall == null) return execution.search(query); diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java index 415ebd7871c..2f9e81c1607 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java @@ -27,9 +27,12 @@ import java.util.Set; * will be returned when attribute prefetch filling is requested.</p> * * @author bratseth + * @deprecated use {@link com.yahoo.search.searchchain.testutil.DocumentSourceSearcher} */ @SuppressWarnings({"rawtypes"}) +@Deprecated // TODO: Remove on Vespa 7 public class DocumentSourceSearcher extends Searcher { + // as for the SuppressWarnings annotation above, we are inside // com.yahoo.prelude, this is old stuff, really no point firing off those // warnings here... @@ -38,7 +41,6 @@ public class DocumentSourceSearcher extends Searcher { private Map<Query, Result> completelyFilledResults = new HashMap<>(); private Map<Query, Result> attributeFilledResults = new HashMap<>(); private Map<Query, Result> unFilledResults = new HashMap<>(); - //private Result defaultUnfilledResult; /** Time (in ms) at which the index of this searcher was last modified */ long editionTimeStamp=0; @@ -101,11 +103,11 @@ public class DocumentSourceSearcher extends Searcher { } /** - * Returns a query clone which has offset and hits set to null. This is used by access to + * Returns a query clone which has source, offset and hits set to null. This is used by access to * the maps using the query as key to achieve lookup independent of offset/hits value */ - private com.yahoo.search.Query getQueryKeyClone(com.yahoo.search.Query query) { - com.yahoo.search.Query key=query.clone(); + private Query getQueryKeyClone(Query query) { + Query key = query.clone(); key.setWindow(0,0); key.getModel().setSources(""); return key; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java index 71e54c810c2..21fa8962da4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java @@ -19,10 +19,9 @@ import java.util.Map; /** - * A searcher which does parametrized collapsing. Based on - * SiteCollapsingSearcher. Deprecated - use grouping. + * A searcher which does parametrized collapsing. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ @SuppressWarnings("deprecation") @After(PhaseNames.RAW_QUERY) @@ -174,17 +173,18 @@ public class FieldCollapsingSearcher extends Searcher { } if (knownCollapses.containsKey(collapseId)) { - int numHitsThisField = knownCollapses.get(collapseId).intValue(); + int numHitsThisField = knownCollapses.get(collapseId); if (numHitsThisField < collapseSize) { result.hits().add(hit); ++numHitsThisField; - knownCollapses.put(collapseId, Integer.valueOf(numHitsThisField)); + knownCollapses.put(collapseId, numHitsThisField); } } else { - knownCollapses.put(collapseId, Integer.valueOf(1)); + knownCollapses.put(collapseId, 1); result.hits().add(hit); } } } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java index c18f3d49da3..2330ca2382a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java @@ -16,16 +16,17 @@ import java.util.Iterator; /** * Save the query in the incoming state to a meta hit in the result. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ - public class JSONDebugSearcher extends Searcher { + public static final String JSON_FIELD = "JSON field: "; public static final String STRUCT_FIELD = "Structured data field (as json): "; public static final String FEATURE_FIELD = "Feature data field (as json): "; private static CompoundName PROPERTYNAME = new CompoundName("dumpjson"); + @Override public Result search(com.yahoo.search.Query query, Execution execution) { Result r = execution.search(query); String propertyName = query.properties().getString(PROPERTYNAME); @@ -53,4 +54,5 @@ public class JSONDebugSearcher extends Searcher { } return r; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java index ca87c0c1d46..5c56379efc0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java @@ -78,13 +78,13 @@ public class JuniperSearcher extends Searcher { @Override public void fill(Result result, String summaryClass, Execution execution) { Result workResult = result; - final int worstCase = workResult.getHitCount(); - final List<Hit> hits = new ArrayList<>(worstCase); - for (final Iterator<Hit> i = workResult.hits().deepIterator(); i.hasNext();) { - final Hit sniffHit = i.next(); + int worstCase = workResult.getHitCount(); + List<Hit> hits = new ArrayList<>(worstCase); + for (Iterator<Hit> i = workResult.hits().deepIterator(); i.hasNext();) { + Hit sniffHit = i.next(); if ( ! (sniffHit instanceof FastHit)) continue; - final FastHit hit = (FastHit) sniffHit; + FastHit hit = (FastHit) sniffHit; if (hit.isFilled(summaryClass)) continue; hits.add(hit); @@ -105,54 +105,46 @@ public class JuniperSearcher extends Searcher { Object searchDefinitionField = hit.getField(MAGIC_FIELD); if (searchDefinitionField == null) continue; - String searchDefinitionName = searchDefinitionField.toString(); - - // TODO: Switch to iterate over indexes in the outer loop: - //for (Index index : indexFacts.getIndexes(searchDefinitionName())) { - // if (index.getDynamicSummary() || index.getHighlightSummary()) { - // insertTags(hit.buildHitField(index.getName(), true, true), bolding, index.getDynamicSummary()); - // } - //} - for (String fieldName : hit.fields().keySet()) { - Index index = indexFacts.getIndex(fieldName, searchDefinitionName); - if (index.getDynamicSummary() || index.getHighlightSummary()) - insertTags(hit.buildHitField(fieldName, true, true), bolding, index.getDynamicSummary()); + + for (Index index : indexFacts.getIndexes(searchDefinitionField.toString())) { + if (index.getDynamicSummary() || index.getHighlightSummary()) { + HitField fieldValue = hit.buildHitField(index.getName(), true, true); + if (fieldValue != null) + insertTags(fieldValue, bolding, index.getDynamicSummary()); + } } } } - private void insertTags(final HitField oldProperty, final boolean bolding, final boolean dynteaser) { + private void insertTags(HitField oldProperty, boolean bolding, boolean dynteaser) { boolean insideHighlight = false; - for (final ListIterator<FieldPart> i = oldProperty.listIterator(); i.hasNext();) { - final FieldPart f = i.next(); - if (f instanceof SeparatorFieldPart) { + for (ListIterator<FieldPart> i = oldProperty.listIterator(); i.hasNext();) { + FieldPart f = i.next(); + if (f instanceof SeparatorFieldPart) setSeparatorString(bolding, (SeparatorFieldPart) f); - } - if (f.isFinal()) { - continue; - } + if (f.isFinal()) continue; - final String toQuote = f.getContent(); + String toQuote = f.getContent(); List<FieldPart> newFieldParts = null; int previous = 0; for (int j = 0; j < toQuote.length(); j++) { - final char key = toQuote.charAt(j); + char key = toQuote.charAt(j); switch (key) { - case RAW_HIGHLIGHT_CHAR: - newFieldParts = initFieldParts(newFieldParts); - addBolding(bolding, insideHighlight, f, toQuote, newFieldParts, previous, j); - previous = j + 1; - insideHighlight = !insideHighlight; - break; - case RAW_SEPARATOR_CHAR: - newFieldParts = initFieldParts(newFieldParts); - addSeparator(bolding, dynteaser, f, toQuote, newFieldParts, - previous, j); - previous = j + 1; - break; - default: - // no action - break; + case RAW_HIGHLIGHT_CHAR: + newFieldParts = initFieldParts(newFieldParts); + addBolding(bolding, insideHighlight, f, toQuote, newFieldParts, previous, j); + previous = j + 1; + insideHighlight = !insideHighlight; + break; + case RAW_SEPARATOR_CHAR: + newFieldParts = initFieldParts(newFieldParts); + addSeparator(bolding, dynteaser, f, toQuote, newFieldParts, + previous, j); + previous = j + 1; + break; + default: + // no action + break; } } if (previous > 0 && previous < toQuote.length()) { @@ -160,37 +152,30 @@ public class JuniperSearcher extends Searcher { } if (newFieldParts != null) { i.remove(); - for (final Iterator<FieldPart> j = newFieldParts.iterator(); j.hasNext();) { + for (Iterator<FieldPart> j = newFieldParts.iterator(); j.hasNext();) { i.add(j.next()); } } } } - private void setSeparatorString(final boolean bolding,final SeparatorFieldPart f) { - if (bolding) { + private void setSeparatorString(boolean bolding, SeparatorFieldPart f) { + if (bolding) f.setContent(separatorTag); - } else { + else f.setContent(ELLIPSIS); - } } - private void addSeparator(final boolean bolding, final boolean dynteaser, - final FieldPart f, final String toQuote, - final List<FieldPart> newFieldParts, final int previous, final int j) { - if (previous != j) { + private void addSeparator(boolean bolding, boolean dynteaser, FieldPart f, String toQuote, + List<FieldPart> newFieldParts, int previous, int j) { + if (previous != j) newFieldParts.add(new StringFieldPart(toQuote.substring(previous, j), f.isToken())); - } - if (dynteaser) { - final FieldPart s = (bolding ? new SeparatorFieldPart(separatorTag) : new SeparatorFieldPart(ELLIPSIS)); - newFieldParts.add(s); - } + if (dynteaser) + newFieldParts.add(bolding ? new SeparatorFieldPart(separatorTag) : new SeparatorFieldPart(ELLIPSIS)); } - private void addBolding(final boolean bolding, - final boolean insideHighlight, final FieldPart f, - final String toQuote, final List<FieldPart> newFieldParts, - final int previous, final int j) { + private void addBolding(boolean bolding, boolean insideHighlight, FieldPart f, String toQuote, + List<FieldPart> newFieldParts, int previous, int j) { if (previous != j) { newFieldParts.add(new StringFieldPart(toQuote.substring(previous, j), f.isToken())); } @@ -209,9 +194,8 @@ public class JuniperSearcher extends Searcher { } private List<FieldPart> initFieldParts(List<FieldPart> newFieldParts) { - if (newFieldParts == null) { + if (newFieldParts == null) newFieldParts = new ArrayList<>(); - } return newFieldParts; } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java index c47af9e32da..3b2fd596cfa 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java @@ -17,7 +17,7 @@ import java.util.*; * * <p> For each group, the desired number of hits can be specified. </p> * - * @author tonytv + * @author tonytv */ public class MultipleResultsSearcher extends Searcher { diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java index 33667349397..43717ecf6cd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java @@ -34,6 +34,7 @@ import com.yahoo.prelude.Location; @Before(PhaseNames.TRANSFORMED_QUERY) @Provides(PosSearcher.POSITION_PARSING) public class PosSearcher extends Searcher { + public static final String POSITION_PARSING = "PositionParsing"; private static final CompoundName posBb = new CompoundName("pos.bb"); @@ -52,7 +53,7 @@ public class PosSearcher extends Searcher { public final static double km2deg = 1000.000 * 180.0 / (Math.PI * 6356752.0); public final static double mi2deg = 1609.344 * 180.0 / (Math.PI * 6356752.0); - + @Override public Result search(Query query, Execution execution) { String bb = query.properties().getString(posBb); String ll = query.properties().getString(posLl); @@ -92,9 +93,8 @@ public class PosSearcher extends Searcher { } } catch (IllegalArgumentException e) { - // System.err.println("error: "+e); - return new Result(query, ErrorMessage.createInvalidQueryParameter( - "Error in pos parameters: " + Exceptions.toMessageString(e))); + return new Result(query, ErrorMessage.createInvalidQueryParameter("Error in pos parameters: " + + Exceptions.toMessageString(e))); } // and finally: query.getRanking().setLocation(loc); @@ -102,8 +102,8 @@ public class PosSearcher extends Searcher { } private void handleGeoCircle(Query query, String ll, Location target) { - double ewCoord = 0; - double nsCoord = 0; + double ewCoord; + double nsCoord; try { DegreesParser parsed = new DegreesParser(ll); ewCoord = parsed.longitude; @@ -111,9 +111,9 @@ public class PosSearcher extends Searcher { } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Unable to parse lat/long string '" +ll + "'", e); } - String radius = query.properties().getString(posRadius); - double radiusdegrees = 0.0; + String radius = query.properties().getString(posRadius); + double radiusdegrees; if (radius == null) { radiusdegrees = 50.0 * km2deg; } else if (radius.endsWith("km")) { @@ -133,8 +133,8 @@ public class PosSearcher extends Searcher { private void handleXyCircle(Query query, String xy, Location target) { - int xcoord = 0; - int ycoord = 0; + int xcoord; + int ycoord; // parse xy int semipos = xy.indexOf(';'); if (semipos > 0 && semipos < xy.length()) { @@ -143,8 +143,9 @@ public class PosSearcher extends Searcher { } else { throw new IllegalArgumentException("pos.xy must be in the format 'digits;digits' but was: '"+xy+"'"); } + String radius = query.properties().getString(posRadius); - int radiusUnits = 0; + int radiusUnits; if (radius == null) { radiusUnits = 5000; } else if (radius.endsWith("km")) { @@ -165,7 +166,6 @@ public class PosSearcher extends Searcher { target.setXyCircle(xcoord, ycoord, radiusUnits); } - private static void parseBoundingBox(String bb, Location target) { BoundingBoxParser parser = new BoundingBoxParser(bb); target.setBoundingBox(parser.n, parser.s, parser.e, parser.w); diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java index 81b948682df..32efcde6feb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java @@ -11,19 +11,20 @@ import com.yahoo.search.searchchain.Execution; /** * Save the query in the incoming state to a meta hit in the result. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated do not use */ - +@Deprecated // TODO: Remove on Vespa 7 public class QuerySnapshotSearcher extends Searcher { public Result search(Query query, Execution execution) { Query q = query.clone(); Result r = execution.search(query); - Hit h = new Hit("meta:querysnapshot", new Relevance( - Double.POSITIVE_INFINITY)); + Hit h = new Hit("meta:querysnapshot", new Relevance(Double.POSITIVE_INFINITY)); h.setMeta(true); h.setField("query", q); r.hits().add(h); return r; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java index 4e604dcd226..558521a7a8d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java @@ -10,8 +10,10 @@ import com.yahoo.search.searchchain.Execution; /** * Ensures hits is 1000 or less and offset is 1000 or less. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated do not use */ +@Deprecated // TODO: Remove on Vespa 7 public class QueryValidatingSearcher extends Searcher { public Result search(Query query, Execution execution) { diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java index d4cad7f1246..5dcc533fb1f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java @@ -35,6 +35,7 @@ public class QuotingSearcher extends Searcher { } private static class QuoteTable { + private final int lowerUncachedBound; private final int upperUncachedBound; private final Map<Character, String> quoteMap; @@ -50,12 +51,10 @@ public class QuotingSearcher extends Searcher { boolean newIsEmpty = true; Map<Character, String> newQuoteMap = new HashMap<>(); for (Iterator<?> i = config.character().iterator(); i.hasNext(); ) { - QrQuotetableConfig.Character character - = (QrQuotetableConfig.Character)i.next(); + QrQuotetableConfig.Character character = (QrQuotetableConfig.Character)i.next(); if (character.ordinal() > 256) { newIsEmpty = false; - newQuoteMap.put(new Character((char)character.ordinal()), - character.quoting()); + newQuoteMap.put(new Character((char)character.ordinal()), character.quoting()); newUseMap = true; if (minOrd == 0 || character.ordinal() < minOrd) minOrd = character.ordinal(); @@ -64,8 +63,7 @@ public class QuotingSearcher extends Searcher { } else { newIsEmpty = false; - newLowerTable[character.ordinal()] - = character.quoting(); + newLowerTable[character.ordinal()] = character.quoting(); } } lowerUncachedBound = minOrd; @@ -75,22 +73,19 @@ public class QuotingSearcher extends Searcher { isEmpty = newIsEmpty; lowerTable = newLowerTable; } + public String get(char c) { - if (isEmpty) - return null; + if (isEmpty) return null; + int ord = (int)c; if (ord < 256) { return lowerTable[ord]; } else { - if ((!useMap) || ord < lowerUncachedBound - || ord > upperUncachedBound) - { + if ((!useMap) || ord < lowerUncachedBound || ord > upperUncachedBound) return null; - } - else { + else return quoteMap.get(new Character(c)); - } } } public boolean isEmpty() { @@ -107,35 +102,29 @@ public class QuotingSearcher extends Searcher { Result result = execution.search(query); execution.fill(result); QuoteTable translations = getQuoteTable(); - if (translations == null || translations.isEmpty()) { - return result; - } + if (translations == null || translations.isEmpty()) return result; + for (Iterator<Hit> i = result.hits().deepIterator(); i.hasNext(); ) { Hit h = i.next(); - if (h instanceof FastHit) { - quoteProperties((FastHit)h, translations); - } + if (h instanceof FastHit) + quoteFields((FastHit) h, translations); } return result; } - private void quoteProperties(FastHit hit, QuoteTable translations) { - for (Iterator<?> i = ((Set<?>) hit.fields().keySet()).iterator(); i.hasNext(); ) { - String propertyName = (String) i.next(); - Object entry = hit.getField(propertyName); - if (entry == null) { - continue; - } - Class<? extends Object> propertyType = entry.getClass(); - if (propertyType.equals(HitField.class)) { - quoteField((HitField) entry, translations); - } else if (propertyType.equals(String.class)) { - quoteProperty(hit, propertyName, (String)entry, translations); + private void quoteFields(FastHit hit, QuoteTable translations) { + hit.forEachField((fieldName, fieldValue) -> { + if (fieldValue != null) { + Class<?> fieldType = fieldValue.getClass(); + if (fieldType.equals(HitField.class)) + quoteField((HitField) fieldValue, translations); + else if (fieldType.equals(String.class)) + quoteField(hit, fieldName, (String) fieldValue, translations); } - } + }); } - private void quoteProperty(Hit hit, String fieldname, String toQuote, QuoteTable translations) { + private void quoteField(Hit hit, String fieldname, String toQuote, QuoteTable translations) { List<FieldPart> l = translate(toQuote, translations, true); if (l != null) { HitField hf = new HitField(fieldname, toQuote); @@ -144,13 +133,11 @@ public class QuotingSearcher extends Searcher { } } - private void quoteField(HitField field, QuoteTable translations) { for (ListIterator<FieldPart> i = field.listIterator(); i.hasNext(); ) { FieldPart f = i.next(); - if (!f.isFinal()) { - List<FieldPart> newFieldParts = translate(f.getContent(), translations, - f.isToken()); + if ( ! f.isFinal()) { + List<FieldPart> newFieldParts = translate(f.getContent(), translations, f.isToken()); if (newFieldParts != null) { i.remove(); for (Iterator<FieldPart> j = newFieldParts.iterator(); j.hasNext(); ) { @@ -161,33 +148,24 @@ public class QuotingSearcher extends Searcher { } } - private List<FieldPart> translate(String toQuote, QuoteTable translations, - boolean isToken) { + private List<FieldPart> translate(String toQuote, QuoteTable translations, boolean isToken) { List<FieldPart> newFieldParts = null; int lastIdx = 0; for (int i = 0; i < toQuote.length(); i++) { String quote = translations.get(toQuote.charAt(i)); if (quote != null) { - if (newFieldParts == null) { + if (newFieldParts == null) newFieldParts = new ArrayList<>(); - } - if (lastIdx != i) { - newFieldParts.add( - new StringFieldPart(toQuote.substring(lastIdx, i), - isToken)); - } + if (lastIdx != i) + newFieldParts.add(new StringFieldPart(toQuote.substring(lastIdx, i), isToken)); String initContent = Character.toString(toQuote.charAt(i)); - newFieldParts.add(new ImmutableFieldPart(initContent, - quote, - isToken)); + newFieldParts.add(new ImmutableFieldPart(initContent, quote, isToken)); lastIdx = i+1; } } - if (lastIdx > 0 && lastIdx < toQuote.length()) { - newFieldParts.add( - new StringFieldPart(toQuote.substring(lastIdx), - isToken)); - } + if (lastIdx > 0 && lastIdx < toQuote.length()) + newFieldParts.add(new StringFieldPart(toQuote.substring(lastIdx), isToken)); return newFieldParts; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java index 2e2c73b6707..9b6f5926b61 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java @@ -20,7 +20,7 @@ import java.util.Collection; /** * Checks that predicate queries don't use values outside the defined upper/lower bounds. * - * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a> + * @author Magnar Nedland */ @After(BooleanSearcher.PREDICATE) public class ValidatePredicateSearcher extends Searcher { @@ -78,4 +78,5 @@ public class ValidatePredicateSearcher extends Searcher { @Override public void onExit() {} } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 9de1a5e2a2d..bd61de7f783 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -10,6 +10,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.metrics.simple.MetricSettings; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.result.Coverage; @@ -207,7 +208,8 @@ public class StatisticsSearcher extends Searcher { * 2) Add response time to total response time (time from entry to return) * 3) ..... */ - public Result search(com.yahoo.search.Query query, Execution execution) { + @Override + public Result search(Query query, Execution execution) { if (query.properties().getBoolean(IGNORE_QUERY,false)) { return execution.search(query); } diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/Context.java b/container-search/src/main/java/com/yahoo/prelude/templates/Context.java index 7a904ea014c..7989f35e77b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/Context.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/Context.java @@ -9,7 +9,10 @@ import com.yahoo.text.XML; * A set of variable bindings for template rendering * * @author bratseth + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public abstract class Context { private boolean xmlEscape = true; diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java b/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java index f14d8ddf319..dab80580f61 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java @@ -13,7 +13,10 @@ import java.util.Set; * Defines formatting options used with special kinds of hits. * * @author laboisse + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public class FormattingOptions { public static final String DEFAULT_TYPE_ATTRIBUTE_NAME = "type"; diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java b/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java index 037d1a77d5c..4d1daa97306 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java @@ -17,7 +17,10 @@ import java.util.Set; * A context providing all the fields of a hit, and falls back to MapContext behavior for all other keys. * * @author tonytv + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public class HitContext extends Context { private final Hit hit; diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java b/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java index 49c5ffa6e78..84d97b71f60 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java @@ -6,7 +6,12 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; -/** A context having a map as secondary storage */ +/** + * A context having a map as secondary storage + * @deprecated use a Renderer instead + */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public class MapContext extends Context { private Map<String, Object> map = new LinkedHashMap<>(); diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java index a24fd623e4d..83118ec66ad 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java @@ -14,7 +14,10 @@ import java.io.Writer; * This is a variant of the tiled template set - see that class for details. * * @author bratseth + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public class PageTemplateSet extends TiledTemplateSet { public PageTemplateSet() { diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java b/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java index 31e133d22d5..a639a6b97ec 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java @@ -21,8 +21,10 @@ import java.util.Iterator; * Renders a search result using the old templates API. * * @author tonytv + * @deprecated do not use */ @SuppressWarnings({ "rawtypes", "deprecation", "unchecked" }) +@Deprecated // TODO: Remove on Vespa 7 public final class SearchRendererAdaptor extends Renderer { private final LogExceptionUserTemplateDelegator templates; diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/Template.java b/container-search/src/main/java/com/yahoo/prelude/templates/Template.java index 63bd3214b17..3d00be9d05b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/Template.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/Template.java @@ -10,7 +10,10 @@ import java.io.Writer; * template mechanism by subclassing this. * * @author bratseth + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public abstract class Template<T extends Writer> { /** @@ -19,8 +22,7 @@ public abstract class Template<T extends Writer> { * @param context the context to evaluate in * @param writer the writer to render to */ - public abstract void render(Context context,T writer) - throws java.io.IOException; + public abstract void render(Context context,T writer) throws java.io.IOException; /** diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java index 6bba0f620ee..91bc33e3e2a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java @@ -70,7 +70,10 @@ import java.util.stream.Collectors; * * @author bratseth * @author laboisse + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public class TiledTemplateSet extends DefaultTemplateSet { private FormattingOptions hitOptionsForProvider; diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 1c6e92ebffc..e9e4e34727c 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -230,7 +230,7 @@ public class SearchHandler extends LoggingRequestHandler { return (e.getCause() instanceof IllegalArgumentException) ? invalidParameterResponse(request, e) : illegalQueryResponse(request, e); - } catch (RuntimeException e) { // Make sure we generate a valid XML response even on unexpected errors + } catch (RuntimeException e) { // Make sure we generate a valid response even on unexpected errors log.log(Level.WARNING, "Failed handling " + request, e); return internalServerErrorResponse(request, e); } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java index 2768a546cd0..399ff6194c8 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java @@ -40,8 +40,8 @@ public class NGramSearcher extends Searcher { private final CharacterClasses characterClasses; public NGramSearcher(Linguistics linguistics) { - gramSplitter= linguistics.getGramSplitter(); - characterClasses= linguistics.getCharacterClasses(); + gramSplitter = linguistics.getGramSplitter(); + characterClasses = linguistics.getCharacterClasses(); } @Override @@ -54,7 +54,7 @@ public class NGramSearcher extends Searcher { if (rewritten) query.trace("Rewritten to n-gram matching",true,2); - Result result=execution.search(query); + Result result = execution.search(query); recombineNGrams(result.hits().deepIterator(), session); return result; } @@ -160,10 +160,11 @@ public class NGramSearcher extends Searcher { if (hit.isMeta()) continue; Object sddocname = hit.getField(Hit.SDDOCNAME_FIELD); if (sddocname == null) return; - for (String fieldName : hit.fieldKeys()) { // TODO: Iterate over indexes instead - Index index = session.getIndex(fieldName, sddocname.toString()); + for (Index index : session.getIndexes(sddocname.toString())) { if (index.isNGram() && (index.getHighlightSummary() || index.getDynamicSummary())) { - hit.setField(fieldName, recombineNGramsField(hit.getField(fieldName), index.getGramSize())); + Object fieldValue = hit.getField(index.getName()); + if (fieldValue != null) + hit.setField(index.getName(), recombineNGramsField(fieldValue, index.getGramSize())); } } } diff --git a/container-search/src/main/java/com/yahoo/search/result/Templating.java b/container-search/src/main/java/com/yahoo/search/result/Templating.java index 47f40f3c7f5..9e191a1219c 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Templating.java +++ b/container-search/src/main/java/com/yahoo/search/result/Templating.java @@ -15,8 +15,10 @@ import com.yahoo.search.query.Presentation; * Helper methods and data store for result attributes geared towards result * rendering and presentation. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated do not use */ +@Deprecated // TODO: Remove on Vespa 7 public class Templating { private final Result result; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java index 56a6a702962..f4973ba4239 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java @@ -28,7 +28,7 @@ import com.yahoo.search.searchchain.Execution; * Any field in the configured hits which has a name starting by attribute * will be returned when attribute prefetch filling is requested.</p> * - * @author bratseth + * @author bratseth */ public class DocumentSourceSearcher extends Searcher { @@ -85,7 +85,6 @@ public class DocumentSourceSearcher extends Searcher { private void addDefaultResults() { Query q = new Query("?query=default"); Result r = new Result(q); - // These four used to assign collapseId 1,2,3,4 - re-add that if needed r.hits().add(new Hit("http://default-1.html", 0)); r.hits().add(new Hit("http://default-2.html", 0)); r.hits().add(new Hit("http://default-3.html", 0)); @@ -97,8 +96,7 @@ public class DocumentSourceSearcher extends Searcher { @Override public Result search(Query query, Execution execution) { queryCount++; - Result r; - r = unFilledResults.get(getQueryKeyClone(query)); + Result r = unFilledResults.get(getQueryKeyClone(query)); if (r == null) { r = defaultFilledResult.clone(); } else { @@ -111,12 +109,13 @@ public class DocumentSourceSearcher extends Searcher { } /** - * Returns a query clone which has offset and hits set to null. This is used by access to + * Returns a query clone which has sourcr, offset and hits set to null. This is used by access to * the maps using the query as key to achieve lookup independent of offset/hits value */ private Query getQueryKeyClone(Query query) { - Query key=query.clone(); + Query key = query.clone(); key.setWindow(0,0); + key.getModel().setSources(""); return key; } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 283a70c478b..4c5dfaabed7 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -30,6 +30,7 @@ import static com.yahoo.search.yql.YqlParser.PREFIX; import static com.yahoo.search.yql.YqlParser.RANGE; import static com.yahoo.search.yql.YqlParser.RANK; import static com.yahoo.search.yql.YqlParser.RANKED; +import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT; import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD; import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE; import static com.yahoo.search.yql.YqlParser.STEM; @@ -79,6 +80,7 @@ import com.yahoo.prelude.query.PrefixItem; import com.yahoo.prelude.query.RangeItem; import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SameElementItem; import com.yahoo.prelude.query.SegmentingRule; import com.yahoo.prelude.query.Substring; import com.yahoo.prelude.query.SubstringItem; @@ -106,8 +108,7 @@ public class VespaSerializer { // TODO refactor, too much copy/paste private static class AndSegmentSerializer extends Serializer { - private static void serializeWords(StringBuilder destination, - AndSegmentItem segment) { + private static void serializeWords(StringBuilder destination, AndSegmentItem segment) { for (int i = 0; i < segment.getItemCount(); ++i) { if (i > 0) { destination.append(", "); @@ -115,28 +116,23 @@ public class VespaSerializer { Item current = segment.getItem(i); if (current instanceof WordItem) { destination.append('"'); - escape(((WordItem) current).getIndexedString(), destination) - .append('"'); + escape(((WordItem) current).getIndexedString(), destination).append('"'); } else { - throw new IllegalArgumentException( - "Serializing of " - + current.getClass().getSimpleName() + throw new IllegalArgumentException("Serializing of " + current.getClass().getSimpleName() + " in segment AND expressions not implemented, please report this as a bug."); } } } @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { return serialize(destination, item, true); } - static boolean serialize(StringBuilder destination, Item item, - boolean includeField) { + static boolean serialize(StringBuilder destination, Item item, boolean includeField) { AndSegmentItem phrase = (AndSegmentItem) item; Substring origin = phrase.getOrigin(); String image; @@ -154,13 +150,11 @@ public class VespaSerializer { } if (includeField) { - destination.append(normalizeIndexName(phrase.getIndexName())) - .append(" contains "); + destination.append(normalizeIndexName(phrase.getIndexName())).append(" contains "); } destination.append("([{"); serializeOrigin(destination, image, offset, length); - destination.append(", \"").append(AND_SEGMENTING) - .append("\": true"); + destination.append(", \"").append(AND_SEGMENTING).append("\": true"); destination.append("}]"); destination.append(PHRASE).append('('); serializeWords(destination, phrase); @@ -189,13 +183,11 @@ public class VespaSerializer { private static class DotProductSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { - serializeWeightedSetContents(destination, DOT_PRODUCT, - (WeightedSetItem) item); + serializeWeightedSetContents(destination, DOT_PRODUCT, (WeightedSetItem) item); return false; } @@ -203,8 +195,7 @@ public class VespaSerializer { private static class EquivSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -240,8 +231,7 @@ public class VespaSerializer { private static class NearSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -304,8 +294,7 @@ public class VespaSerializer { private static class NullSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -319,31 +308,22 @@ public class VespaSerializer { private static class NumberSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { IntItem intItem = (IntItem) item; - if (intItem.getFromLimit().number() - .equals(intItem.getToLimit().number())) { - destination.append(normalizeIndexName(intItem.getIndexName())) - .append(" = "); - annotatedNumberImage(intItem, intItem.getFromLimit().number() - .toString(), destination); + if (intItem.getFromLimit().number().equals(intItem.getToLimit().number())) { + destination.append(normalizeIndexName(intItem.getIndexName())).append(" = "); + annotatedNumberImage(intItem, intItem.getFromLimit().number().toString(), destination); } else if (intItem.getFromLimit().isInfinite()) { destination.append(normalizeIndexName(intItem.getIndexName())); - destination.append(intItem.getToLimit().isInclusive() ? " <= " - : " < "); - annotatedNumberImage(intItem, intItem.getToLimit().number() - .toString(), destination); + destination.append(intItem.getToLimit().isInclusive() ? " <= " : " < "); + annotatedNumberImage(intItem, intItem.getToLimit().number().toString(), destination); } else if (intItem.getToLimit().isInfinite()) { destination.append(normalizeIndexName(intItem.getIndexName())); - destination - .append(intItem.getFromLimit().isInclusive() ? " >= " - : " > "); - annotatedNumberImage(intItem, intItem.getFromLimit().number() - .toString(), destination); + destination.append(intItem.getFromLimit().isInclusive() ? " >= " : " > "); + annotatedNumberImage(intItem, intItem.getFromLimit().number().toString(), destination); } else { serializeAsRange(destination, intItem); } @@ -358,21 +338,17 @@ public class VespaSerializer { int initLen; if (leftOpen && rightOpen) { - boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_OPEN - + "\""; + boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_OPEN + "\""; } else if (leftOpen) { - boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" - + BOUNDS_LEFT_OPEN + "\""; + boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_LEFT_OPEN + "\""; } else if (rightOpen) { - boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" - + BOUNDS_RIGHT_OPEN + "\""; + boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_RIGHT_OPEN + "\""; } if (annotations.length() > 0 || boundsAnnotation.length() > 0) { destination.append("[{"); } initLen = destination.length(); if (annotations.length() > 0) { - destination.append(annotations); } comma(destination, initLen); @@ -389,8 +365,7 @@ public class VespaSerializer { .append(")"); } - private void annotatedNumberImage(IntItem item, String rawNumber, - StringBuilder image) { + private void annotatedNumberImage(IntItem item, String rawNumber, StringBuilder image) { String annotations = leafAnnotations(item); if (annotations.length() > 0) { @@ -430,16 +405,14 @@ public class VespaSerializer { private static class RegExpSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { RegExpItem regexp = (RegExpItem) item; String annotations = leafAnnotations(regexp); - destination.append(normalizeIndexName(regexp.getIndexName())).append( - " matches "); + destination.append(normalizeIndexName(regexp.getIndexName())).append(" matches "); annotatedTerm(destination, regexp, annotations); return false; } @@ -498,8 +471,7 @@ public class VespaSerializer { private static class PhraseSegmentSerializer extends Serializer { - private static void serializeWords(StringBuilder destination, - PhraseSegmentItem segment) { + private static void serializeWords(StringBuilder destination, PhraseSegmentItem segment) { for (int i = 0; i < segment.getItemCount(); ++i) { if (i > 0) { destination.append(", "); @@ -507,20 +479,16 @@ public class VespaSerializer { Item current = segment.getItem(i); if (current instanceof WordItem) { destination.append('"'); - escape(((WordItem) current).getIndexedString(), destination) - .append('"'); + escape(((WordItem) current).getIndexedString(), destination).append('"'); } else { - throw new IllegalArgumentException( - "Serializing of " - + current.getClass().getSimpleName() - + " in phrases not implemented, please report this as a bug."); + throw new IllegalArgumentException("Serializing of " + current.getClass().getSimpleName() + + " in phrases not implemented, please report this as a bug."); } } } @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -535,8 +503,7 @@ public class VespaSerializer { int length; if (includeField) { - destination.append(normalizeIndexName(phrase.getIndexName())) - .append(" contains "); + destination.append(normalizeIndexName(phrase.getIndexName())).append(" contains "); } if (origin == null) { image = phrase.getRawWord(); @@ -555,8 +522,7 @@ public class VespaSerializer { destination.append(", ").append(annotations); } if (phrase.getSegmentingRule() == SegmentingRule.BOOLEAN_AND) { - destination.append(", ").append('"').append(AND_SEGMENTING) - .append("\": true"); + destination.append(", ").append('"').append(AND_SEGMENTING).append("\": true"); } destination.append("}]"); destination.append(PHRASE).append('('); @@ -568,16 +534,14 @@ public class VespaSerializer { private static class PhraseSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { return serialize(destination, item, true); } - static boolean serialize(StringBuilder destination, Item item, - boolean includeField) { + static boolean serialize(StringBuilder destination, Item item, boolean includeField) { PhraseItem phrase = (PhraseItem) item; String annotations = leafAnnotations(phrase); @@ -598,11 +562,9 @@ public class VespaSerializer { } Item current = phrase.getItem(i); if (current instanceof WordItem) { - WordSerializer.serializeWordWithoutIndex(destination, - current); + WordSerializer.serializeWordWithoutIndex(destination, current); } else if (current instanceof PhraseSegmentItem) { - PhraseSegmentSerializer.serialize(destination, current, - false); + PhraseSegmentSerializer.serialize(destination, current, false); } else if (current instanceof WordAlternativesItem) { WordAlternativesSerializer.serialize(destination, (WordAlternativesItem) current, false); } else { @@ -621,16 +583,60 @@ public class VespaSerializer { } - private static class PredicateQuerySerializer extends Serializer { + private static class SameElementSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { + void onExit(StringBuilder destination, Item item) { } + + @Override + boolean serialize(StringBuilder destination, Item item) { + return serialize(destination, item, true); } + static boolean serialize(StringBuilder destination, Item item, boolean includeField) { + + SameElementItem phrase = (SameElementItem) item; + String annotations = leafAnnotations(phrase); + + if (includeField) { + destination.append(normalizeIndexName(phrase.getIndexName())).append(" contains "); + + } + if (annotations.length() > 0) { + destination.append("([{").append(annotations).append("}]"); + } + + destination.append(SAME_ELEMENT).append('('); + for (int i = 0; i < phrase.getItemCount(); ++i) { + if (i > 0) { + destination.append(", "); + } + Item current = phrase.getItem(i); + if (current instanceof WordItem) { + new WordSerializer().serialize(destination, current); + + } else { + throw new IllegalArgumentException( + "Serializing of " + current.getClass().getSimpleName() + + " in same_element is not implemented, please report this as a bug."); + } + } + destination.append(')'); + if (annotations.length() > 0) { + destination.append(')'); + } + return false; + } + + } + + private static class PredicateQuerySerializer extends Serializer { + @Override + void onExit(StringBuilder destination, Item item) { } + @Override boolean serialize(StringBuilder destination, Item item) { PredicateQueryItem pItem = (PredicateQueryItem) item; - destination.append("predicate(").append(pItem.getIndexName()) - .append(','); + destination.append("predicate(").append(pItem.getIndexName()).append(','); appendFeatures(destination, pItem.getFeatures()); destination.append(','); appendFeatures(destination, pItem.getRangeFeatures()); @@ -638,8 +644,7 @@ public class VespaSerializer { return false; } - private void appendFeatures(StringBuilder destination, - Collection<? extends PredicateQueryItem.EntryBase> features) { + private void appendFeatures(StringBuilder destination, Collection<? extends PredicateQueryItem.EntryBase> features) { if (features.isEmpty()) { destination.append('0'); // Workaround for empty maps. return; @@ -651,8 +656,7 @@ public class VespaSerializer { destination.append(','); } if (entry.getSubQueryBitmap() != PredicateQueryItem.ALL_SUB_QUERIES) { - destination.append("\"0x").append( - Long.toHexString(entry.getSubQueryBitmap())); + destination.append("\"0x").append(Long.toHexString(entry.getSubQueryBitmap())); destination.append("\":{"); appendKeyValue(destination, entry); destination.append('}'); @@ -664,19 +668,16 @@ public class VespaSerializer { destination.append('}'); } - private void appendKeyValue(StringBuilder destination, - PredicateQueryItem.EntryBase entry) { + private void appendKeyValue(StringBuilder destination, PredicateQueryItem.EntryBase entry) { destination.append('"'); escape(entry.getKey(), destination); destination.append("\":"); if (entry instanceof PredicateQueryItem.Entry) { destination.append('"'); - escape(((PredicateQueryItem.Entry) entry).getValue(), - destination); + escape(((PredicateQueryItem.Entry) entry).getValue(), destination); destination.append('"'); } else { - destination.append(((PredicateQueryItem.RangeEntry) entry) - .getValue()); + destination.append(((PredicateQueryItem.RangeEntry) entry).getValue()); destination.append('L'); } } @@ -685,8 +686,7 @@ public class VespaSerializer { private static class RangeSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -737,8 +737,7 @@ public class VespaSerializer { private static class WordAlternativesSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -800,10 +799,8 @@ public class VespaSerializer { abstract void onExit(StringBuilder destination, Item item); String separator(Deque<SerializerWrapper> state) { - throw new UnsupportedOperationException( - "Having several items for this query operator serializer, " - + this.getClass().getSimpleName() - + ", not yet implemented."); + throw new UnsupportedOperationException("Having several items for this query operator serializer, " + + this.getClass().getSimpleName() + ", not yet implemented."); } abstract boolean serialize(StringBuilder destination, Item item); @@ -822,8 +819,7 @@ public class VespaSerializer { } - private static final class TokenComparator implements - Comparator<Entry<Object, Integer>> { + private static final class TokenComparator implements Comparator<Entry<Object, Integer>> { @SuppressWarnings({ "rawtypes", "unchecked" }) @Override @@ -858,8 +854,7 @@ public class VespaSerializer { Serializer doIt = dispatch.get(item.getClass()); if (doIt == null) { - throw new IllegalArgumentException(item.getClass() - + " not supported for YQL+ marshalling."); + throw new IllegalArgumentException(item.getClass() + " not supported for YQL+ marshalling."); } if (state.peekFirst() != null && state.peekFirst().subItems > 0) { @@ -878,9 +873,7 @@ public class VespaSerializer { @Override boolean serialize(StringBuilder destination, Item item) { - serializeWeightedSetContents(destination, WAND, - (WeightedSetItem) item, - specificAnnotations((WandItem) item)); + serializeWeightedSetContents(destination, WAND, (WeightedSetItem) item, specificAnnotations((WandItem) item)); return false; } @@ -890,18 +883,15 @@ public class VespaSerializer { double scoreThreshold = w.getScoreThreshold(); double thresholdBoostFactor = w.getThresholdBoostFactor(); if (targetNumHits != 10) { - annotations.append('"').append(TARGET_NUM_HITS).append("\": ") - .append(targetNumHits); + annotations.append('"').append(TARGET_NUM_HITS).append("\": ").append(targetNumHits); } if (scoreThreshold != 0) { comma(annotations, 0); - annotations.append('"').append(SCORE_THRESHOLD).append("\": ") - .append(scoreThreshold); + annotations.append('"').append(SCORE_THRESHOLD).append("\": ").append(scoreThreshold); } if (thresholdBoostFactor != 1) { comma(annotations, 0); - annotations.append('"').append(THRESHOLD_BOOST_FACTOR) - .append("\": ").append(thresholdBoostFactor); + annotations.append('"').append(THRESHOLD_BOOST_FACTOR).append("\": ").append(thresholdBoostFactor); } return annotations.toString(); } @@ -963,8 +953,7 @@ public class VespaSerializer { @Override boolean serialize(StringBuilder destination, Item item) { - serializeWeightedSetContents(destination, WEIGHTED_SET, - (WeightedSetItem) item); + serializeWeightedSetContents(destination, WEIGHTED_SET, (WeightedSetItem) item); return false; } @@ -981,14 +970,12 @@ public class VespaSerializer { WordItem w = (WordItem) item; StringBuilder wordAnnotations = getAllAnnotations(w); - destination.append(normalizeIndexName(w.getIndexName())).append( - " contains "); + destination.append(normalizeIndexName(w.getIndexName())).append(" contains "); VespaSerializer.annotatedTerm(destination, w, wordAnnotations.toString()); return false; } - static void serializeWordWithoutIndex(StringBuilder destination, - Item item) { + static void serializeWordWithoutIndex(StringBuilder destination, Item item) { WordItem w = (WordItem) item; StringBuilder wordAnnotations = getAllAnnotations(w); @@ -996,8 +983,7 @@ public class VespaSerializer { } private static StringBuilder getAllAnnotations(WordItem w) { - StringBuilder wordAnnotations = new StringBuilder( - WordSerializer.wordAnnotations(w)); + StringBuilder wordAnnotations = new StringBuilder(WordSerializer.wordAnnotations(w)); String leafAnnotations = leafAnnotations(w); if (leafAnnotations.length() > 0) { @@ -1034,15 +1020,12 @@ public class VespaSerializer { length = origin.end - origin.start; } - if (!image.substring(offset, offset + length).equals( - item.getIndexedString())) { - VespaSerializer.serializeOrigin(annotation, image, offset, - length); + if (!image.substring(offset, offset + length).equals(item.getIndexedString())) { + VespaSerializer.serializeOrigin(annotation, image, offset, length); } if (usePositionData != true) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(USE_POSITION_DATA) - .append("\": false"); + annotation.append('"').append(USE_POSITION_DATA).append("\": false"); } if (stemmed == true) { VespaSerializer.comma(annotation, initLen); @@ -1050,8 +1033,7 @@ public class VespaSerializer { } if (lowercased == true) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(NORMALIZE_CASE) - .append("\": false"); + annotation.append('"').append(NORMALIZE_CASE).append("\": false"); } if (accentDrop == false) { VespaSerializer.comma(annotation, initLen); @@ -1059,13 +1041,11 @@ public class VespaSerializer { } if (andSegmenting == SegmentingRule.BOOLEAN_AND) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(AND_SEGMENTING) - .append("\": true"); + annotation.append('"').append(AND_SEGMENTING).append("\": true"); } if (!isFromQuery) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(IMPLICIT_TRANSFORMS) - .append("\": false"); + annotation.append('"').append(IMPLICIT_TRANSFORMS).append("\": false"); } if (prefix) { VespaSerializer.comma(annotation, initLen); @@ -1106,9 +1086,9 @@ public class VespaSerializer { dispatchBuilder.put(ONearItem.class, new ONearSerializer()); dispatchBuilder.put(OrItem.class, new OrSerializer()); dispatchBuilder.put(PhraseItem.class, new PhraseSerializer()); + dispatchBuilder.put(SameElementItem.class, new SameElementSerializer()); dispatchBuilder.put(PhraseSegmentItem.class, new PhraseSegmentSerializer()); - dispatchBuilder.put(PredicateQueryItem.class, - new PredicateQuerySerializer()); + dispatchBuilder.put(PredicateQueryItem.class, new PredicateQuerySerializer()); dispatchBuilder.put(PrefixItem.class, new WordSerializer()); // gotcha dispatchBuilder.put(WordAlternativesItem.class, new WordAlternativesSerializer()); dispatchBuilder.put(RangeItem.class, new RangeSerializer()); @@ -1225,24 +1205,20 @@ public class VespaSerializer { return out.toString(); } - private static void serializeWeightedSetContents(StringBuilder destination, - String opName, WeightedSetItem weightedSet) { + private static void serializeWeightedSetContents(StringBuilder destination, String opName, + WeightedSetItem weightedSet) { serializeWeightedSetContents(destination, opName, weightedSet, ""); } - private static void serializeWeightedSetContents( - StringBuilder destination, - String opName, WeightedSetItem weightedSet, - String optionalAnnotations) { + private static void serializeWeightedSetContents(StringBuilder destination, String opName, + WeightedSetItem weightedSet, String optionalAnnotations) { addAnnotations(destination, weightedSet, optionalAnnotations); destination.append(opName).append('(') .append(normalizeIndexName(weightedSet.getIndexName())) .append(", {"); int initLen = destination.length(); - List<Entry<Object, Integer>> tokens = new ArrayList<>( - weightedSet.getNumTokens()); - for (Iterator<Entry<Object, Integer>> i = weightedSet.getTokens(); i - .hasNext();) { + List<Entry<Object, Integer>> tokens = new ArrayList<>(weightedSet.getNumTokens()); + for (Iterator<Entry<Object, Integer>> i = weightedSet.getTokens(); i.hasNext();) { tokens.add(i.next()); } Collections.sort(tokens, tokenComparator); @@ -1255,9 +1231,8 @@ public class VespaSerializer { destination.append("})"); } - private static void addAnnotations( - StringBuilder destination, - WeightedSetItem weightedSet, String optionalAnnotations) { + private static void addAnnotations(StringBuilder destination, WeightedSetItem weightedSet, + String optionalAnnotations) { int preAnnotationValueLen; int incomingLen = destination.length(); String annotations = leafAnnotations(weightedSet); @@ -1303,13 +1278,11 @@ public class VespaSerializer { } if (item.hasExplicitSignificance()) { comma(annotation, initLen); - annotation.append('"').append(SIGNIFICANCE).append("\": ") - .append(significance); + annotation.append('"').append(SIGNIFICANCE).append("\": ").append(significance); } if (uniqueId != 0) { comma(annotation, initLen); - annotation.append('"').append(UNIQUE_ID).append("\": ") - .append(uniqueId); + annotation.append('"').append(UNIQUE_ID).append("\": ").append(uniqueId); } } { @@ -1335,25 +1308,21 @@ public class VespaSerializer { } if (weight != 100) { comma(annotation, initLen); - annotation.append('"').append(WEIGHT).append("\": ") - .append(weight); + annotation.append('"').append(WEIGHT).append("\": ").append(weight); } } if (item instanceof IntItem) { int hitLimit = ((IntItem) item).getHitLimit(); if (hitLimit != 0) { comma(annotation, initLen); - annotation.append('"').append(HIT_LIMIT).append("\": ") - .append(hitLimit); + annotation.append('"').append(HIT_LIMIT).append("\": ").append(hitLimit); } } return annotation.toString(); } - private static void serializeOrigin(StringBuilder destination, - String image, int offset, int length) { - destination.append('"').append(ORIGIN).append("\": {\"") - .append(ORIGIN_ORIGINAL).append("\": \""); + private static void serializeOrigin(StringBuilder destination, String image, int offset, int length) { + destination.append('"').append(ORIGIN).append("\": {\"").append(ORIGIN_ORIGINAL).append("\": \""); escape(image, destination); destination.append("\", \"").append(ORIGIN_OFFSET).append("\": ") .append(offset).append(", \"").append(ORIGIN_LENGTH) diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 259719571be..292bb6d0f5a 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -47,6 +47,7 @@ import com.yahoo.prelude.query.PrefixItem; import com.yahoo.prelude.query.RangeItem; import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SameElementItem; import com.yahoo.prelude.query.SegmentItem; import com.yahoo.prelude.query.SegmentingRule; import com.yahoo.prelude.query.Substring; @@ -161,6 +162,7 @@ public class YqlParser implements Parser { static final String RANGE = "range"; static final String RANKED = "ranked"; static final String RANK = "rank"; + static final String SAME_ELEMENT = "sameElement"; static final String SCORE_THRESHOLD = "scoreThreshold"; static final String SIGNIFICANCE = "significance"; static final String STEM = "stem"; @@ -533,6 +535,17 @@ public class YqlParser implements Parser { } @NonNull + private Item instantiateSameElementItem(String field, OperatorNode<ExpressionOperator> ast) { + assertHasFunctionName(ast, SAME_ELEMENT); + + SameElementItem sameElement = new SameElementItem(field); + for (OperatorNode<ExpressionOperator> word : ast.<List<OperatorNode<ExpressionOperator>>> getArgument(1)) { + sameElement.addItem(buildTermSearch(word)); + } + return leafStyleSettings(ast, sameElement); + } + + @NonNull private Item instantiatePhraseItem(String field, OperatorNode<ExpressionOperator> ast) { assertHasFunctionName(ast, PHRASE); @@ -1183,6 +1196,8 @@ public class YqlParser implements Parser { List<String> names = ast.getArgument(0); Preconditions.checkArgument(names.size() == 1, "Expected 1 name, got %s.", names.size()); switch (names.get(0)) { + case SAME_ELEMENT: + return instantiateSameElementItem(field, ast); case PHRASE: return instantiatePhraseItem(field, ast); case NEAR: @@ -1194,7 +1209,7 @@ public class YqlParser implements Parser { case ALTERNATIVES: return instantiateWordAlternativesItem(field, ast); default: - throw newUnexpectedArgumentException(names.get(0), EQUIV, NEAR, ONEAR, PHRASE); + throw newUnexpectedArgumentException(names.get(0), EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java new file mode 100644 index 00000000000..ff3ca53319f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java @@ -0,0 +1,35 @@ +package com.yahoo.prelude.query.test; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.SameElementItem; +import com.yahoo.prelude.query.WordItem; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class SameElementItemTestCase { + @Test + public void testAddItem() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new WordItem("b", "f1")); + s.addItem(new WordItem("c", "f2")); + s.addItem(new WordItem("d", "f3")); + assertEquals("structa:{f1:b f2:c f3:d}", s.toString()); + } + @Test(expected = IllegalArgumentException.class) + public void requireAllChildrenHaveStructMemberNameSet() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new WordItem("b", "f1")); + s.addItem(new WordItem("c")); + } + @Test(expected = IllegalArgumentException.class) + public void requireAllChildrenHaveNonEmptyTerm() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new WordItem("", "f2")); + } + @Test(expected = IllegalArgumentException.class) + public void requireAllChildrenAreTermItems() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new AndItem()); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java index f71c0803ce8..9f4a12d24e6 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java @@ -28,9 +28,9 @@ import java.util.List; import java.util.Map; /** - * Tests conversion of juniper highlighting to XML + * Tests juniper highlighting * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class JuniperSearcherTestCase { diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java index ee735104caa..691f877dfba 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java @@ -15,9 +15,9 @@ import com.yahoo.search.Result; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.hitfield.HitField; import com.yahoo.search.Searcher; -import com.yahoo.prelude.searcher.DocumentSourceSearcher; import com.yahoo.prelude.searcher.QuotingSearcher; import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import org.junit.Test; import java.util.ArrayList; @@ -55,7 +55,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", "smith & jones"); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); assertEquals("smith & jones", check.hits().get(0).getField("title").toString()); assertTrue(check.hits().get(0).fields().containsKey("title")); @@ -75,7 +75,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", "&smith &jo& nes"); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); assertEquals("&smith &jo& nes", check.hits().get(0).getField("title").toString()); assertTrue(check.hits().get(0).fields().containsKey("title")); @@ -95,7 +95,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", new HitField("title", "&smith &jo& nes")); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); assertEquals("&smith &jo& nes", check.hits().get(0).getField("title").toString()); assertTrue(check.hits().get(0).fields().containsKey("title")); @@ -116,7 +116,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", Integer.valueOf(42)); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); // should not quote non-string properties assertEquals(Integer.valueOf(42), check.hits().get(0).getField("title")); diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java index 99b7eeff9a0..a3f7ff12319 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java @@ -46,32 +46,32 @@ public class NGramSearcherTestCase { @Before public void setUp() { - searcher=new NGramSearcher(new SimpleLinguistics()); - indexFacts=new IndexFacts(); + searcher = new NGramSearcher(new SimpleLinguistics()); + indexFacts = new IndexFacts(); - Index defaultIndex=new Index("default"); - defaultIndex.setNGram(true,3); + Index defaultIndex = new Index("default"); + defaultIndex.setNGram(true, 3); defaultIndex.setDynamicSummary(true); - indexFacts.addIndex("default",defaultIndex); + indexFacts.addIndex("default", defaultIndex); - Index test=new Index("test"); + Index test = new Index("test"); test.setHighlightSummary(true); - indexFacts.addIndex("default",test); + indexFacts.addIndex("default", test); - Index gram2=new Index("gram2"); - gram2.setNGram(true,2); + Index gram2 = new Index("gram2"); + gram2.setNGram(true, 2); gram2.setDynamicSummary(true); - indexFacts.addIndex("default",gram2); + indexFacts.addIndex("default", gram2); - Index gram3=new Index("gram3"); - gram3.setNGram(true,3); + Index gram3 = new Index("gram3"); + gram3.setNGram(true, 3); gram3.setHighlightSummary(true); - indexFacts.addIndex("default",gram3); + indexFacts.addIndex("default", gram3); - Index gram14=new Index("gram14"); - gram14.setNGram(true,14); + Index gram14 = new Index("gram14"); + gram14.setNGram(true, 14); gram14.setDynamicSummary(true); - indexFacts.addIndex("default",gram14); + indexFacts.addIndex("default", gram14); } private IndexFacts getMixedSetup() { diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index d6bcdf3195f..6984a8537ef 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.search.yql; import static org.junit.Assert.*; +import com.yahoo.prelude.query.SameElementItem; import com.yahoo.search.Query; import com.yahoo.search.grouping.Continuation; import com.yahoo.search.grouping.GroupingRequest; @@ -218,6 +219,15 @@ public class VespaSerializerTestCase { } @Test + public final void testSameElement() { + SameElementItem sameElement = new SameElementItem("ss"); + sameElement.addItem(new WordItem("a", "f1")); + sameElement.addItem(new WordItem("b", "f2")); + assertEquals("ss:{f1:a f2:b}", sameElement.toString()); + assertEquals("ss contains sameElement(f1 contains ([{\"implicitTransforms\": false}]\"a\"), f2 contains ([{\"implicitTransforms\": false}]\"b\"))", VespaSerializer.serialize(sameElement)); + + } + @Test public final void testAnnotatedAndSegment() { AndSegmentItem andSegment = new AndSegmentItem("abc", true, false); andSegment.addItem(new WordItem("a", "indexNamePlaceholder")); @@ -307,6 +317,7 @@ public class VespaSerializerTestCase { assertEquals("default contains \"nalle\"", q); } + @Test public final void testLongAndNot() { NotItem item = new NotItem(); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 75b39f26625..9d04dc545e1 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -261,6 +261,12 @@ public class YqlParserTestCase { } @Test + public void testSameElement() { + assertParse("select foo from bar where baz contains sameElement(f1 contains \"a\", f2 contains \"b\");", + "baz:{f1:a f2:b}"); + } + + @Test public void testPhrase() { assertParse("select foo from bar where baz contains phrase(\"a\", \"b\");", "baz:\"a b\""); diff --git a/container-test-jars/jersey-resources/pom.xml b/container-test-jars/jersey-resources/pom.xml index 33a3f03a962..2b6761e6411 100644 --- a/container-test-jars/jersey-resources/pom.xml +++ b/container-test-jars/jersey-resources/pom.xml @@ -16,11 +16,6 @@ <dependencies> <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - </dependency> - - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespa_jersey2</artifactId> <version>${project.version}</version> @@ -31,27 +26,6 @@ <build> <plugins> <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>add-source</goal> - <goal>compile</goal> - <goal>testCompile</goal> - </goals> - </execution> - </executions> - <configuration> - <args> - <arg>-unchecked</arg> - <arg>-deprecation</arg> - <arg>-feature</arg> - </args> - </configuration> - </plugin> - - <plugin> <groupId>com.yahoo.vespa</groupId> <artifactId>bundle-plugin</artifactId> <version>${project.version}</version> diff --git a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResource.java b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResource.java new file mode 100644 index 00000000000..59095d05567 --- /dev/null +++ b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResource.java @@ -0,0 +1,12 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.test.jars.jersey.resources; + +import javax.ws.rs.Path; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +@Path("bundle-plugin-test/test-resource") +public class TestResource extends TestResourceBase { +} diff --git a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.java b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.java new file mode 100644 index 00000000000..c3724723252 --- /dev/null +++ b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.java @@ -0,0 +1,22 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.test.jars.jersey.resources; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.Produces; +import javax.ws.rs.GET; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class TestResourceBase { + @GET + @Produces({MediaType.TEXT_PLAIN}) + public String get() { + return content(getClass()); + } + + public static String content(Class<? extends TestResourceBase> clazz) { + return "Response from " + clazz.getName(); + } +} diff --git a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/nestedpackage1/NestedTestResource1.scala b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage1/NestedTestResource1.java index 440f2f45cea..ab1c1f8f229 100644 --- a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/nestedpackage1/NestedTestResource1.scala +++ b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage1/NestedTestResource1.java @@ -1,12 +1,13 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources.nestedpackage1 +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.test.jars.jersey.resources.nestedpackage1; -import javax.ws.rs.Path +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase +import javax.ws.rs.Path; /** - * @author tonytv + * @author Tony Vaagenes */ @Path("bundle-plugin-test/nested-test-resource1") -class NestedTestResource1 extends TestResourceBase +public class NestedTestResource1 extends TestResourceBase { +} diff --git a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/nestedpackage2/NestedTestResource2.scala b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage2/NestedTestResource2.java index 5fa354dd647..0dfc9e1938b 100644 --- a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/nestedpackage2/NestedTestResource2.scala +++ b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage2/NestedTestResource2.java @@ -1,12 +1,13 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources.nestedpackage2 +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.test.jars.jersey.resources.nestedpackage2; -import javax.ws.rs.Path +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase +import javax.ws.rs.Path; /** - * @author tonytv + * @author Tony Vaagenes */ @Path("bundle-plugin-test/nested-test-resource2") -class NestedTestResource2 extends TestResourceBase +public class NestedTestResource2 extends TestResourceBase { +} diff --git a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResource.scala b/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResource.scala deleted file mode 100644 index b73a8735789..00000000000 --- a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResource.scala +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources - -import javax.ws.rs.Path - -/** - * @author tonytv - */ -@Path("bundle-plugin-test/test-resource") -class TestResource extends TestResourceBase diff --git a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.scala b/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.scala deleted file mode 100644 index 5ccd89b30ac..00000000000 --- a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.scala +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources - -import javax.ws.rs.core.MediaType -import javax.ws.rs.{Produces, GET} - -import scala.reflect.ClassTag - -/** - * @author tonytv - */ -class TestResourceBase { - @GET - @Produces(Array(MediaType.TEXT_PLAIN)) - def get() = TestResourceBase.content(getClass) -} - -object TestResourceBase { - def content(clazz: Class[_ <: TestResourceBase]): String = - "Response from " + clazz.getName - - def content[T <: TestResourceBase](implicit classTag: ClassTag[T]): String = { - val clazz = classTag.runtimeClass.asInstanceOf[Class[_ <: TestResourceBase]] - content(clazz) - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index db749713483..8b0dc35e16b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -341,16 +341,16 @@ public class ApplicationController { /** Deploy a system application to given zone */ public void deploy(SystemApplication application, ZoneId zone, Version version) { - if (!application.hasApplicationPackage()) { + if (application.hasApplicationPackage()) { + ApplicationPackage applicationPackage = new ApplicationPackage( + artifactRepository.getSystemApplicationPackage(application.id(), zone, version) + ); + DeployOptions options = withVersion(version, DeployOptions.none()); + deploy(application.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet()); + } else { // Deploy by calling node repository directly configServer().nodeRepository().upgrade(zone, application.nodeType(), version); - return; } - ApplicationPackage applicationPackage = new ApplicationPackage( - artifactRepository.getSystemApplicationPackage(application.id(), zone, version) - ); - DeployOptions options = withVersion(version, DeployOptions.none()); - deploy(application.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet()); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 4ed45af5e66..ca642d0fb2c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -82,9 +82,9 @@ public class ApplicationList { return listOf(list.stream().filter(application -> application.change().isPresent())); } - /** Returns the subset of applications which are currently not deploying a change */ - public ApplicationList notDeploying() { - return listOf(list.stream().filter(application -> ! application.change().isPresent())); + /** Returns the subset of applications which are currently really not deploying a change */ + public ApplicationList notDeployingAt(Instant now) { + return listOf(list.stream().filter(application -> ! application.changeAt(now).isPresent())); } /** Returns the subset of applications which currently does not have any failing jobs */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index 280d4c7f7e4..1fa579684de 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -59,6 +59,11 @@ public final class Change { /** Returns an instance representing no change */ public static Change empty() { return empty; } + /** Returns a version of this change which replaces or adds this platform change */ + public Change with(Version platformVersion) { + return new Change(Optional.of(platformVersion), application); + } + /** Returns a version of this change which replaces or adds this application change */ public Change with(ApplicationVersion applicationVersion) { return new Change(platform, Optional.of(applicationVersion)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index d3daa68741e..1e96f33c275 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -12,17 +12,23 @@ import java.util.List; * * @author mpolden */ -public enum SystemApplication { +public enum SystemApplication { + // Note that the enum declaration order decides the upgrade order + configServerHost(ApplicationId.from("hosted-vespa", "configserver-host", "default"), NodeType.confighost), + proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), - zone(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy); + zone(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy, + configServerHost, proxyHost, configServer); private final ApplicationId id; private final NodeType nodeType; + private final List<SystemApplication> dependencies; - SystemApplication(ApplicationId id, NodeType nodeType) { + SystemApplication(ApplicationId id, NodeType nodeType, SystemApplication... dependencies) { this.id = id; this.nodeType = nodeType; + this.dependencies = Arrays.asList(dependencies); } public ApplicationId id() { @@ -34,7 +40,10 @@ public enum SystemApplication { return nodeType; } - /** Returns whether this system application has its own application package */ + /** Returns the system applications that should upgrade before this */ + public List<SystemApplication> dependencies() { return dependencies; } + + /** Returns whether this system application has an application package */ public boolean hasApplicationPackage() { return nodeType == NodeType.proxy; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 017d1062ab3..e902206ad8b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -97,7 +97,7 @@ public class DeploymentTrigger { * trigger next. */ public void notifyOfCompletion(JobReport report) { - log.log(LogLevel.INFO, String.format("Notified of %s for %s of %s (%d).", + log.log(LogLevel.INFO, String.format("Notified of %s for %s of %s (%d)", report.jobError().map(e -> e.toString() + " error") .orElse("success"), report.jobType(), @@ -216,7 +216,7 @@ public class DeploymentTrigger { */ public void triggerChange(ApplicationId applicationId, Change change) { applications().lockOrThrow(applicationId, application -> { - if (application.change().isPresent() && ! application.deploymentJobs().hasFailures()) + if (application.changeAt(controller.clock().instant()).isPresent() && ! application.deploymentJobs().hasFailures()) throw new IllegalArgumentException("Could not start " + change + " on " + application + ": " + application.change() + " is already in progress"); application = application.withChange(change); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 93103213274..7154cf7d600 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -58,7 +58,7 @@ public class ControllerMaintenance extends AbstractComponent { deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService); - systemUpgrader = new SystemUpgrader(controller, maintenanceInterval, jobControl); + systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); } public Upgrader upgrader() { return upgrader; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java index ff3d44d5a8c..63361e5dcc4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java @@ -22,7 +22,7 @@ public class OutstandingChangeDeployer extends Maintainer { protected void maintain() { ApplicationList applications = ApplicationList.from(controller().applications().asList()).notPullRequest(); for (Application application : applications.asList()) { - if (!application.change().isPresent() && application.outstandingChange().isPresent()) { + if ( ! application.change().isPresent() && application.outstandingChange().isPresent()) { controller().applications().deploymentTrigger().triggerChange(application.id(), application.outstandingChange()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index d7fc71e4b08..516cd52d710 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.time.Duration; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -37,51 +38,63 @@ public class SystemUpgrader extends Maintainer { if (!target.isPresent()) { return; } + // TODO: Change to SystemApplication.all() once host applications support upgrade + try { + deploy(Arrays.asList(SystemApplication.configServer, SystemApplication.zone), target.get()); + } catch (Exception e) { + log.log(Level.WARNING, "Failed to upgrade system. Retrying in " + maintenanceInterval(), e); + } + } + + /** Deploy a list of system applications until they converge on the given version */ + private void deploy(List<SystemApplication> applications, Version target) { for (List<ZoneId> zones : controller().zoneRegistry().upgradePolicy().asList()) { - // The order here is important. Config servers should always upgrade first - if (!deploy(zones, SystemApplication.configServer, target.get())) { - break; + boolean converged = true; + for (ZoneId zone : zones) { + for (SystemApplication application : applications) { + boolean dependenciesConverged = application.dependencies().stream() + .filter(applications::contains) // TODO: Remove when all() is used. + .allMatch(dependency -> currentVersion(zone, dependency.id()).equals(target)); + if (dependenciesConverged) { + deploy(target, application, zone); + } + converged &= currentVersion(zone, application.id()).equals(target); + } } - if (!deploy(zones, SystemApplication.zone, target.get())) { + if (!converged) { break; } } } - /** Deploy application on given version. Returns true when all allocated nodes are on requested version */ - private boolean deploy(List<ZoneId> zones, SystemApplication application, Version version) { - boolean completed = true; - for (ZoneId zone : zones) { - if (!wantedVersion(zone, application.id()).equals(version)) { - log.info(String.format("Deploying %s version %s in %s", application.id(), version, zone)); - controller().applications().deploy(application, zone, version); - } - completed = completed && currentVersion(zone, application.id()).equals(version); + /** Deploy application on given version idempotently */ + private void deploy(Version target, SystemApplication application, ZoneId zone) { + if (!wantedVersion(zone, application.id(), target).equals(target)) { + log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone)); + controller().applications().deploy(application, zone, target); } - return completed; } - private Version wantedVersion(ZoneId zone, ApplicationId application) { - return minVersion(zone, application, Node::wantedVersion); + private Version wantedVersion(ZoneId zone, ApplicationId application, Version defaultVersion) { + return minVersion(zone, application, Node::wantedVersion).orElse(defaultVersion); } private Version currentVersion(ZoneId zone, ApplicationId application) { - return minVersion(zone, application, Node::currentVersion); + return minVersion(zone, application, Node::currentVersion).orElse(Version.emptyVersion); } - private Version minVersion(ZoneId zone, ApplicationId application, Function<Node, Version> versionField) { + private Optional<Version> minVersion(ZoneId zone, ApplicationId application, Function<Node, Version> versionField) { try { return controller().configServer() .nodeRepository() .listOperational(zone, application) .stream() .map(versionField) - .min(Comparator.naturalOrder()) - .orElse(Version.emptyVersion); + .min(Comparator.naturalOrder()); } catch (Exception e) { log.log(Level.WARNING, String.format("Failed to get version for %s in %s: %s", application, zone, Exceptions.toMessageString(e))); - return Version.emptyVersion; + return Optional.empty(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 4692c2fb23a..bd8b8fc8747 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -94,14 +94,14 @@ public class Upgrader extends Maintainer { applications = applications.notPullRequest(); // Pull requests are deployed as separate applications to test then deleted; No need to upgrade applications = applications.hasProductionDeployment(); applications = applications.onLowerVersionThan(version); - applications = applications.notDeploying(); // wait with applications deploying an application change or already upgrading + applications = applications.notDeployingAt(controller().clock().instant()); // wait with applications deploying an application change or already upgrading applications = applications.notFailingOn(version); // try to upgrade only if it hasn't failed on this version applications = applications.canUpgradeAt(controller().clock().instant()); // wait with applications that are currently blocking upgrades applications = applications.byIncreasingDeployedVersion(); // start with lowest versions applications = applications.first(numberOfApplicationsToUpgrade()); // throttle upgrades for (Application application : applications.asList()) { try { - controller().applications().deploymentTrigger().triggerChange(application.id(), Change.of(version)); + controller().applications().deploymentTrigger().triggerChange(application.id(), application.change().with(version)); } catch (IllegalArgumentException e) { log.log(Level.INFO, "Could not trigger change: " + Exceptions.toMessageString(e)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java index e9eec0682f3..99b913709df 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java @@ -54,13 +54,17 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Inject public ConfigServerMock(ZoneRegistryMock zoneRegistry) { - bootstrap(zoneRegistry.zones().all().ids()); + bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.all()); } - public void bootstrap(List<ZoneId> zones) { + public void bootstrap(List<ZoneId> zones, SystemApplication... applications) { + bootstrap(zones, Arrays.asList(applications)); + } + + public void bootstrap(List<ZoneId> zones, List<SystemApplication> applications) { nodeRepository().clear(); for (ZoneId zone : zones) { - for (SystemApplication application : SystemApplication.all()) { + for (SystemApplication application : applications) { List<Node> nodes = IntStream.rangeClosed(1, 3) .mapToObj(i -> new Node( HostName.from("node-" + i + "-" + application.id().application() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 5790ab2a0c8..1ec64f8d478 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import org.junit.Ignore; import org.junit.Test; import java.util.Arrays; @@ -16,6 +17,7 @@ import java.util.List; import java.util.function.Function; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author mpolden @@ -39,7 +41,9 @@ public class SystemUpgraderTest { ); Version version1 = Version.fromString("6.5"); - tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4)); + // Bootstrap a system without host applications + tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4), SystemApplication.configServer, + SystemApplication.zone); // Fail a few nodes. Failed nodes should not affect versions failNodeIn(zone1, SystemApplication.configServer); failNodeIn(zone3, SystemApplication.zone); @@ -68,17 +72,25 @@ public class SystemUpgraderTest { tester.systemUpgrader().maintain(); assertWantedVersion(SystemApplication.zone, version2, zone1); completeUpgrade(SystemApplication.zone, version2, zone1); + assertTrue("Deployed zone application", + tester.configServer().application(SystemApplication.zone.id()).isPresent()); // zone 2, 3 and 4: still targets old version assertWantedVersion(SystemApplication.configServer, version1, zone2, zone3, zone4); assertWantedVersion(SystemApplication.zone, version1, zone2, zone3, zone4); - // zone 2 and 3: zone-config-server upgrades in parallel + // zone 2 and 3: zone-config-server upgrades, first in zone 2, then in zone 3 tester.systemUpgrader().maintain(); assertWantedVersion(SystemApplication.configServer, version2, zone2, zone3); assertWantedVersion(SystemApplication.configServer, version1, zone4); assertWantedVersion(SystemApplication.zone, version1, zone2, zone3, zone4); - completeUpgrade(SystemApplication.configServer, version2, zone2, zone3); + completeUpgrade(SystemApplication.configServer, version2, zone2); + + // zone-application starts upgrading in zone 2, while zone-config-server completes upgrade in zone 3 + tester.systemUpgrader().maintain(); + assertWantedVersion(SystemApplication.zone, version2, zone2); + assertWantedVersion(SystemApplication.zone, version1, zone3); + completeUpgrade(SystemApplication.configServer, version2, zone3); // zone 2 and 3: zone-application upgrades in parallel tester.systemUpgrader().maintain(); @@ -109,6 +121,55 @@ public class SystemUpgraderTest { } @Test + @Ignore // TODO: Unignore once host applications support upgrade + public void upgrade_system_containing_host_applications() { + tester.controllerTester().zoneRegistry().setUpgradePolicy( + UpgradePolicy.create() + .upgrade(zone1) + .upgradeInParallel(zone2, zone3) + .upgrade(zone4) + ); + + Version version1 = Version.fromString("6.5"); + tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4), SystemApplication.all()); + tester.upgradeSystem(version1); + tester.systemUpgrader().maintain(); + assertCurrentVersion(SystemApplication.all(), version1, zone1, zone2, zone3, zone4); + + // Controller upgrades + Version version2 = Version.fromString("6.6"); + tester.upgradeController(version2); + assertEquals(version2, tester.controller().versionStatus().controllerVersion().get().versionNumber()); + + // System upgrades in zone 1: + tester.systemUpgrader().maintain(); + List<SystemApplication> allExceptZone = Arrays.asList(SystemApplication.configServerHost, + SystemApplication.proxyHost, + SystemApplication.configServer); + completeUpgrade(allExceptZone, version2, zone1); + tester.systemUpgrader().maintain(); + completeUpgrade(SystemApplication.zone, version2, zone1); + assertWantedVersion(SystemApplication.all(), version1, zone2, zone3, zone4); + + // zone 2 and 3: + tester.systemUpgrader().maintain(); + completeUpgrade(allExceptZone, version2, zone2, zone3); + tester.systemUpgrader().maintain(); + completeUpgrade(SystemApplication.zone, version2, zone2, zone3); + assertWantedVersion(SystemApplication.all(), version1, zone4); + + // zone 4: + tester.systemUpgrader().maintain(); + completeUpgrade(allExceptZone, version2, zone4); + tester.systemUpgrader().maintain(); + completeUpgrade(SystemApplication.zone, version2, zone4); + + // All done + tester.systemUpgrader().maintain(); + assertWantedVersion(SystemApplication.all(), version2, zone1, zone2, zone3, zone4); + } + + @Test public void never_downgrades_system() { ZoneId zone = ZoneId.from("prod", "eu-west-1"); tester.controllerTester().zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(zone)); @@ -130,6 +191,7 @@ public class SystemUpgraderTest { /** Simulate upgrade of nodes allocated to given application. In a real system this is done by the node itself */ private void completeUpgrade(SystemApplication application, Version version, ZoneId... zones) { + assertWantedVersion(application, version, zones); for (ZoneId zone : zones) { for (Node node : nodeRepository().listOperational(zone, application.id())) { nodeRepository().add(zone, new Node(node.hostname(), node.state(), node.type(), node.owner(), @@ -139,6 +201,10 @@ public class SystemUpgraderTest { } } + private void completeUpgrade(List<SystemApplication> applications, Version version, ZoneId... zones) { + applications.forEach(application -> completeUpgrade(application, version, zones)); + } + private void failNodeIn(ZoneId zone, SystemApplication application) { List<Node> nodes = nodeRepository().list(zone, application.id()); if (nodes.isEmpty()) { @@ -157,11 +223,19 @@ public class SystemUpgraderTest { assertVersion(application.id(), version, Node::currentVersion, zones); } + private void assertWantedVersion(List<SystemApplication> applications, Version version, ZoneId... zones) { + applications.forEach(application -> assertVersion(application.id(), version, Node::wantedVersion, zones)); + } + + private void assertCurrentVersion(List<SystemApplication> applications, Version version, ZoneId... zones) { + applications.forEach(application -> assertVersion(application.id(), version, Node::currentVersion, zones)); + } + private void assertVersion(ApplicationId application, Version version, Function<Node, Version> versionField, ZoneId... zones) { for (ZoneId zone : zones) { for (Node node : nodeRepository().listOperational(zone, application)) { - assertEquals(version, versionField.apply(node)); + assertEquals(application + " version", version, versionField.apply(node)); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 4d2e64d66c6..5d6fb76cacf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -621,9 +621,6 @@ public class UpgraderTest { public void testBlockVersionChangeHalfwayThough() { ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:00:00.00Z")); // Tuesday, 17:00 DeploymentTester tester = new DeploymentTester(new ControllerTester(clock)); - ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), - Duration.ofHours(1), - new JobControl(tester.controllerTester().curator())); Version version = Version.fromString("5.0"); tester.upgradeSystem(version); @@ -936,4 +933,65 @@ public class UpgraderTest { } } + @Test + public void testBlockRevisionChangeHalfwayThoughThenUpgrade() { + ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:00:00.00Z")); // Tuesday, 17:00. + DeploymentTester tester = new DeploymentTester(new ControllerTester(clock)); + + Version version = Version.fromString("5.0"); + tester.upgradeSystem(version); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + // Block upgrades on Tuesday in hours 18 and 19. + .blockChange(true, false, "tue", "18-19", "UTC") + .region("us-west-1") + .region("us-central-1") + .region("us-east-3") + .build(); + + Application app = tester.createAndDeploy("app1", 1, applicationPackage); + + tester.jobCompletion(component).application(app).nextBuildNumber().uploadArtifact(applicationPackage).submit(); + + // Application upgrade starts. + tester.upgrader().maintain(); + tester.triggerUntilQuiescence(); + tester.deployAndNotify(app, applicationPackage, true, systemTest); + tester.deployAndNotify(app, applicationPackage, true, stagingTest); + clock.advance(Duration.ofHours(1)); // Entering block window after prod job is triggered. + tester.deployAndNotify(app, applicationPackage, true, productionUsWest1); + assertTrue(tester.buildService().jobs().isEmpty()); // Next job not triggered due to being in the block window. + + // One hour passes, time is 19:00, still no upgrade. + tester.clock().advance(Duration.ofHours(1)); + tester.triggerUntilQuiescence(); + assertTrue("No jobs scheduled", tester.buildService().jobs().isEmpty()); + + // New version is released and upgrades are started in the two first production zones. + version = Version.fromString("5.1"); + tester.upgradeSystem(version); + tester.deployAndNotify(app, applicationPackage, true, systemTest); + tester.deployAndNotify(app, applicationPackage, true, stagingTest); + tester.deployAndNotify(app, applicationPackage, true, productionUsWest1); + + // Tests for central-1. + tester.deployAndNotify(app, applicationPackage, true, systemTest); + tester.deployAndNotify(app, applicationPackage, true, stagingTest); + + // Another hour pass, time is 20:00 and both revision and version upgrades are now allowed. + tester.clock().advance(Duration.ofHours(1)); + tester.triggerUntilQuiescence(); // Tests that trigger now test the full upgrade, since central-1 is still on old versions. + tester.deployAndNotify(app, applicationPackage, true, productionUsCentral1); // Only upgrade for now. + // west-1 is now fully upgraded, central-1 only has new version, and east-3 has only old versions. + + // These tests were triggered with an upgrade of both version and revision. Since central-1 no longer upgrades version, + // it ignores the initial version of the staging job, and so the current staging job is OK for both zones. + tester.deployAndNotify(app, applicationPackage, true, systemTest); + tester.deployAndNotify(app, applicationPackage, true, stagingTest); + tester.deployAndNotify(app, applicationPackage, true, productionUsCentral1); + tester.deployAndNotify(app, applicationPackage, true, productionUsEast3); + assertTrue("All jobs consumed", tester.buildService().jobs().isEmpty()); + } + } diff --git a/defaults/src/apps/printdefault/CMakeLists.txt b/defaults/src/apps/printdefault/CMakeLists.txt index 03fe38bcef4..2823b90d251 100644 --- a/defaults/src/apps/printdefault/CMakeLists.txt +++ b/defaults/src/apps/printdefault/CMakeLists.txt @@ -7,3 +7,4 @@ vespa_add_executable(defaults_vespa-print-default_app DEPENDS vespadefaults ) +set_source_files_properties(printdefault.cpp PROPERTIES COMPILE_FLAGS "${VTAG_DEFINES}") diff --git a/defaults/src/apps/printdefault/printdefault.cpp b/defaults/src/apps/printdefault/printdefault.cpp index 73e199fb441..33c47dbcb20 100644 --- a/defaults/src/apps/printdefault/printdefault.cpp +++ b/defaults/src/apps/printdefault/printdefault.cpp @@ -8,8 +8,9 @@ int main(int argc, char **argv) { if (argc != 2) { fprintf(stderr, "usage: %s <variable>\n", argv[0]); fprintf(stderr, " variable names are: home, user, hostname, portbase, configservers,\n"); - fprintf(stderr, " configserver_rpc_port, configservers_rpc\n"); - fprintf(stderr, " configservers_http, configsources, configproxy_rpc\n"); + fprintf(stderr, " configserver_rpc_port, configservers_rpc,\n"); + fprintf(stderr, " configservers_http, configsources, configproxy_rpc,\n"); + fprintf(stderr, " version\n"); return 1; } if (strcmp(argv[1], "home") == 0) { @@ -47,6 +48,8 @@ int main(int argc, char **argv) { } else if (strcmp(argv[1], "configproxy_rpc") == 0) { std::string v = vespa::Defaults::vespaConfigProxyRpcAddr(); printf("%s\n", v.c_str()); + } else if (strcmp(argv[1], "version") == 0) { + printf("%s\n", V_TAG_COMPONENT); } else { fprintf(stderr, "Unknown variable '%s'\n", argv[1]); return 1; diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 837ebd873e3..f9cf3472110 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -905,7 +905,7 @@ ArithmeticValueNode::getValue(std::unique_ptr<Value> lval, slval.getValue() + srval.getValue())); } } - //@fallthrough@ + [[fallthrough]]; case SUB: case MUL: case DIV: @@ -995,7 +995,7 @@ ArithmeticValueNode::traceValue(std::unique_ptr<Value> lval, return result; } } - //@fallthrough@ + [[fallthrough]]; case SUB: case MUL: case DIV: diff --git a/fastlib/src/vespa/fastlib/util/base64.cpp b/fastlib/src/vespa/fastlib/util/base64.cpp index 1b7889d8c47..8b9ac45e698 100644 --- a/fastlib/src/vespa/fastlib/util/base64.cpp +++ b/fastlib/src/vespa/fastlib/util/base64.cpp @@ -90,7 +90,7 @@ Fast_Base64::Decode(const char *source, unsigned int length, char *destination) if (symbol != '=' || i == length) return -1; symbol = source[++i]; - //@fallthrough@ + [[fallthrough]]; case 3: for (; i < length; ++i) { symbol = source[i]; if (symbol == '\0') diff --git a/fbench/src/filterfile/filterfile.cpp b/fbench/src/filterfile/filterfile.cpp index ca93b70a046..e9b35de97e0 100644 --- a/fbench/src/filterfile/filterfile.cpp +++ b/fbench/src/filterfile/filterfile.cpp @@ -133,7 +133,7 @@ main(int argc, char** argv) break; case 1: buf[outIdx++] = line[idx]; - //@fallthrough@ + [[fallthrough]]; case 2: if (line[idx++] == '&') state = 0; diff --git a/fsa/src/vespa/fsa/segmenter.cpp b/fsa/src/vespa/fsa/segmenter.cpp index d13249a5ce8..3bcb3f1b489 100644 --- a/fsa/src/vespa/fsa/segmenter.cpp +++ b/fsa/src/vespa/fsa/segmenter.cpp @@ -77,16 +77,16 @@ void Segmenter::Segments::buildSegmentation(Segmenter::SegmentationMethod method switch(method){ case SEGMENTATION_WEIGHTED_BIAS100: bias+=50; - //@fallthrough@ + [[fallthrough]]; case SEGMENTATION_WEIGHTED_BIAS50: bias+=30; - //@fallthrough@ + [[fallthrough]]; case SEGMENTATION_WEIGHTED_BIAS20: bias+=10; - //@fallthrough@ + [[fallthrough]]; case SEGMENTATION_WEIGHTED_BIAS10: bias+=10; - //@fallthrough@ + [[fallthrough]]; case SEGMENTATION_WEIGHTED: bestid=-1; for(i=n_txt;i>=0;i--){ diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index d491daf55dd..c85917c4c7e 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -360,10 +360,11 @@ public class JettyHttpServer extends AbstractServerProvider { return statisticsHandler; } + @SuppressWarnings("deprecation") private GzipHandler newGzipHandler(ServerConfig serverConfig) { GzipHandler gzipHandler = new GzipHandlerWithVaryHeaderFixed(); gzipHandler.setCompressionLevel(serverConfig.responseCompressionLevel()); - gzipHandler.setCheckGzExists(false); + gzipHandler.setCheckGzExists(false); // TODO: will be removed without replacement in Jetty 10 gzipHandler.setIncludedMethods("GET", "POST"); return gzipHandler; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java index be54df31c50..3b17972db6d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java @@ -5,12 +5,15 @@ import com.yahoo.vespa.hosted.dockerapi.DockerImage; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class NodeAttributes { private Optional<Long> restartGeneration = Optional.empty(); private Optional<Long> rebootGeneration = Optional.empty(); private Optional<DockerImage> dockerImage = Optional.empty(); + private Optional<String> vespaVersion = Optional.empty(); private Optional<String> hardwareDivergence = Optional.empty(); public NodeAttributes() { } @@ -30,6 +33,11 @@ public class NodeAttributes { return this; } + public NodeAttributes withVespaVersion(String vespaVersion) { + this.vespaVersion = Optional.of(vespaVersion); + return this; + } + public NodeAttributes withHardwareDivergence(String hardwareDivergence) { this.hardwareDivergence = Optional.of(hardwareDivergence); return this; @@ -48,13 +56,17 @@ public class NodeAttributes { return dockerImage; } + public Optional<String> getVespaVersion() { + return vespaVersion; + } + public Optional<String> getHardwareDivergence() { return hardwareDivergence; } @Override public int hashCode() { - return Objects.hash(restartGeneration, rebootGeneration, dockerImage, hardwareDivergence); + return Objects.hash(restartGeneration, rebootGeneration, dockerImage, vespaVersion, hardwareDivergence); } @Override @@ -67,16 +79,20 @@ public class NodeAttributes { return Objects.equals(restartGeneration, other.restartGeneration) && Objects.equals(rebootGeneration, other.rebootGeneration) && Objects.equals(dockerImage, other.dockerImage) + && Objects.equals(vespaVersion, other.vespaVersion) && Objects.equals(hardwareDivergence, other.hardwareDivergence); } @Override public String toString() { - return "NodeAttributes{" + - "restartGeneration=" + restartGeneration.map(String::valueOf).orElse("") + - ", rebootGeneration=" + rebootGeneration.map(String::valueOf).orElse("") + - ", dockerImage=" + dockerImage.map(DockerImage::asString).orElse("") + - ", hardwareDivergence='" + hardwareDivergence.orElse(null) + "'" + - '}'; + return Stream.of( + restartGeneration.map(gen -> "restartGeneration=" + gen), + rebootGeneration.map(gen -> "rebootGeneration=" + gen), + dockerImage.map(img -> "dockerImage=" + img.asString()), + vespaVersion.map(ver -> "vespaVersion=" + ver), + hardwareDivergence.map(hwDivg -> "hardwareDivergence=" + hwDivg)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.joining(", ", "NodeAttributes{", "}")); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index 71229818975..fb2a1e3f890 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -207,6 +207,7 @@ public class RealNodeRepository implements NodeRepository { node.currentDockerImage = nodeAttributes.getDockerImage().map(DockerImage::asString).orElse(null); node.currentRestartGeneration = nodeAttributes.getRestartGeneration().orElse(null); node.currentRebootGeneration = nodeAttributes.getRebootGeneration().orElse(null); + node.vespaVersion = nodeAttributes.getVespaVersion().orElse(null); node.hardwareDivergence = nodeAttributes.getHardwareDivergence().orElse(null); return node; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 869e59d890b..8c59568c00d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -40,6 +40,7 @@ import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Base64; import java.util.Set; import static java.util.Collections.singleton; @@ -252,7 +253,7 @@ public class AthenzCredentialsMaintainer { idDoc.instanceHostname(), idDoc.createdAt(), idDoc.ipAddresses()); - String rawIdentityDocument = objectMapper.writeValueAsString(identityDocumentPayload); + String rawIdentityDocument = Base64.getEncoder().encodeToString(objectMapper.writeValueAsString(identityDocumentPayload).getBytes()); com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocument payload = new com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocument( rawIdentityDocument, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java index 24736683845..26ca069aceb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java @@ -7,6 +7,10 @@ import java.nio.file.NoSuchFileException; import java.util.Optional; /** + * Utils related to IOException. + * + * todo: replace much of the below with com.yahoo.yolean.Exceptions::uncheck + * * @author hakonhall */ public class IOExceptionUtil { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java index ed315708b51..8987a7d6af3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -55,10 +55,10 @@ public class MultiDockerTest { "DeleteContainerStorage with ContainerName { name=host2 }"); dockerTester.callOrderVerifier.assertInOrder( - "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, hardwareDivergence='null'}", - "updateNodeAttributes with HostName: host2.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image2, hardwareDivergence='null'}", + "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1}", + "updateNodeAttributes with HostName: host2.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image2}", "setNodeState host2.test.yahoo.com to ready", - "updateNodeAttributes with HostName: host3.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, hardwareDivergence='null'}"); + "updateNodeAttributes with HostName: host3.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1}"); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java index 651da7caec5..fb841f63f0c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java @@ -30,7 +30,7 @@ public class RestartTest { // Check that the container is started and NodeRepo has received the PATCH update dockerTester.callOrderVerifier.assertInOrder( "createContainerCommand with DockerImage { imageId=image:1.2.3 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image:1.2.3, hardwareDivergence='null'}"); + "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image:1.2.3}"); wantedRestartGeneration = 2; currentRestartGeneration = 1; diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java index fe09f23d282..fef5a695db6 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java @@ -157,7 +157,7 @@ public class CoreCollector { private Path compressCoredump(Path coredumpPath) throws IOException { if (! coredumpPath.toString().endsWith(".lz4")) { processExecuter.exec( - new String[]{LZ4_PATH, coredumpPath.toString(), coredumpPath.toString() + ".lz4"}); + new String[]{LZ4_PATH, "-f", coredumpPath.toString(), coredumpPath.toString() + ".lz4"}); return coredumpPath; } else { @@ -167,7 +167,7 @@ public class CoreCollector { Path decompressedPath = Paths.get(coredumpPath.toString().replaceFirst("\\.lz4$", "")); Pair<Integer, String> result = processExecuter.exec( - new String[]{LZ4_PATH, "-f", "-d", coredumpPath.toString(), decompressedPath.toString()}); + new String[] {LZ4_PATH, "-f", "-d", coredumpPath.toString(), decompressedPath.toString()}); if (result.getFirst() != 0) { throw new RuntimeException("Failed to decompress file " + coredumpPath + ": " + result); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java index fd9ee237046..31d9a606d91 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -111,6 +111,7 @@ public class NodePatcher { .map(DockerImage::tagAsVersion) .orElse(Version.emptyVersion); return node.with(node.status().withVespaVersion(versionFromImage)); + case "vespaVersion" : case "currentVespaVersion" : return node.with(node.status().withVespaVersion(Version.fromString(asString(value)))); case "failCount" : diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h b/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h index 4be1f00dcbc..d18c4840009 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h +++ b/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h @@ -5,18 +5,12 @@ #include <vespa/searchcommon/common/range.h> #include <vespa/vespalib/stllike/string.h> -namespace search { -namespace fef { - class TermFieldMatchData; -} -namespace queryeval { - class SearchIterator; -} +namespace search::fef { class TermFieldMatchData; } +namespace search::queryeval { class SearchIterator; } +namespace search { class QueryTermBase; } -class QueryTermBase; - -namespace attribute { +namespace search::attribute { class ISearchContext { public: @@ -24,8 +18,8 @@ public: using DocId = uint32_t; private: - virtual bool onCmp(DocId docId, int32_t &weight) const = 0; - virtual bool onCmp(DocId docId) const = 0; + virtual int32_t onFind(DocId docId, int32_t elementId, int32_t &weight) const = 0; + virtual int32_t onFind(DocId docId, int32_t elementId) const = 0; public: virtual ~ISearchContext() {} @@ -57,10 +51,19 @@ public: virtual const QueryTermBase &queryTerm() const = 0; virtual const vespalib::string &attributeName() const = 0; - bool cmp(DocId docId, int32_t &weight) const { return onCmp(docId, weight); } - bool cmp(DocId docId) const { return onCmp(docId); } + int32_t find(DocId docId, int32_t elementId, int32_t &weight) const { return onFind(docId, elementId, weight); } + int32_t find(DocId docId, int32_t elementId) const { return onFind(docId, elementId); } + bool matches(DocId docId, int32_t &weight) const { + weight = 0; + int32_t oneWeight(0); + int32_t firstId = find(docId, 0, oneWeight); + for (int32_t id(firstId); id >= 0; id = find(docId, id + 1, oneWeight)) { + weight += oneWeight; + } + return firstId >= 0; + } + bool matches(DocId doc) const { return find(doc, 0) >= 0; } }; } -} diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp index 068b7f4fa00..e10e6ed539b 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp @@ -12,15 +12,14 @@ using search::QueryTermSimple; using search::fef::TermFieldMatchData; using search::queryeval::SearchIterator; -namespace proton { -namespace documentmetastore { +namespace proton::documentmetastore { namespace { class GidAllSearchIterator : public AttributeIteratorBase { private: - virtual void + void doSeek(uint32_t docId) override { if (_store.validLidFast(docId)) { @@ -28,7 +27,7 @@ private: } } - virtual void + void doUnpack(uint32_t docId) override { _matchData->reset(docId); @@ -37,8 +36,7 @@ private: protected: const DocumentMetaStore & _store; public: - GidAllSearchIterator(TermFieldMatchData *matchData, - const DocumentMetaStore &store) + GidAllSearchIterator(TermFieldMatchData *matchData, const DocumentMetaStore &store) : AttributeIteratorBase(matchData), _store(store) { @@ -79,7 +77,7 @@ class GidSearchIterator : public GidAllSearchIterator private: const GlobalId & _gid; - virtual void + void doSeek(uint32_t docId) override { AttributeVector::DocId lid = 0; @@ -90,9 +88,7 @@ private: } } public: - GidSearchIterator(TermFieldMatchData *matchData, - const DocumentMetaStore &store, - const GlobalId &gid) + GidSearchIterator(TermFieldMatchData *matchData, const DocumentMetaStore &store, const GlobalId &gid) : GidAllSearchIterator(matchData, store), _gid(gid) { @@ -101,23 +97,16 @@ public: } -bool -SearchContext::onCmp(DocId docId, int32_t &weight) const +int32_t +SearchContext::onFind(DocId, int32_t, int32_t &) const { - (void) docId; - (void) weight; - throw vespalib::IllegalStateException( - "The function is not implemented for documentmetastore::SearchContext"); - return false; + throw vespalib::IllegalStateException("The function is not implemented for documentmetastore::SearchContext"); } -bool -SearchContext::onCmp(DocId docId) const +int32_t +SearchContext::onFind(DocId, int32_t ) const { - (void) docId; - throw vespalib::IllegalStateException( - "The function is not implemented for documentmetastore::SearchContext"); - return false; + throw vespalib::IllegalStateException("The function is not implemented for documentmetastore::SearchContext"); } unsigned int @@ -127,15 +116,13 @@ SearchContext::approximateHits() const } SearchIterator::UP -SearchContext::createIterator(TermFieldMatchData *matchData, - bool strict) +SearchContext::createIterator(TermFieldMatchData *matchData, bool strict) { return _isWord - ? SearchIterator::UP(new GidSearchIterator(matchData, getStore(), _gid)) + ? std::make_unique<GidSearchIterator>(matchData, getStore(), _gid) : strict - ? SearchIterator::UP(new GidStrictAllSearchIterator(matchData, - getStore())) - : SearchIterator::UP(new GidAllSearchIterator(matchData, getStore())); + ? std::make_unique<GidStrictAllSearchIterator>(matchData, getStore()) + : std::make_unique<GidAllSearchIterator>(matchData, getStore()); } const DocumentMetaStore & @@ -144,12 +131,10 @@ SearchContext::getStore() const return static_cast<const DocumentMetaStore &>(attribute()); } -SearchContext::SearchContext(QueryTermSimple::UP qTerm, - const DocumentMetaStore &toBeSearched) +SearchContext::SearchContext(QueryTermSimple::UP qTerm, const DocumentMetaStore &toBeSearched) : search::AttributeVector::SearchContext(toBeSearched), _isWord(qTerm->isWord()) { } } -} diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h index afd09be43e9..31c520bedf5 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h @@ -6,8 +6,7 @@ #include <vespa/searchlib/attribute/attributevector.h> #include "documentmetastore.h" -namespace proton { -namespace documentmetastore { +namespace proton::documentmetastore { /** * Search context used to search the document meta store for all valid documents. @@ -15,14 +14,14 @@ namespace documentmetastore { class SearchContext : public search::AttributeVector::SearchContext { private: - typedef search::AttributeVector::DocId DocId; + using DocId = search::AttributeVector::DocId; bool _isWord; document::GlobalId _gid; unsigned int approximateHits() const override; - bool onCmp(DocId docId, int32_t &weight) const override; - bool onCmp(DocId docId) const override; + int32_t onFind(DocId docId, int32_t elemId, int32_t &weight) const override; + int32_t onFind(DocId docId, int32_t elemId) const override; search::queryeval::SearchIterator::UP createIterator(search::fef::TermFieldMatchData *matchData, bool strict) override; @@ -30,10 +29,7 @@ private: const DocumentMetaStore &getStore() const; public: - SearchContext(std::unique_ptr<search::QueryTermSimple> qTerm, - const DocumentMetaStore &toBeSearched); + SearchContext(std::unique_ptr<search::QueryTermSimple> qTerm, const DocumentMetaStore &toBeSearched); }; -} // namespace documentmetastore -} // namespace proton - +} diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp index e8e16ffcc98..0a02824e77a 100644 --- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp +++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp @@ -301,24 +301,24 @@ TEST_F("Strict iterator handles seek outside of LID space", ArrayValueFixture) { EXPECT_TRUE(iter->isAtEnd()); } -TEST_F("cmp() performs GID mapping and forwards to target attribute", SingleValueFixture) { +TEST_F("matches() performs GID mapping and forwards to target attribute", SingleValueFixture) { auto ctx = f.create_context(word_term("5678")); - EXPECT_FALSE(ctx->cmp(DocId(2))); - EXPECT_TRUE(ctx->cmp(DocId(3))); - EXPECT_FALSE(ctx->cmp(DocId(4))); - EXPECT_TRUE(ctx->cmp(DocId(5))); + EXPECT_FALSE(ctx->matches(DocId(2))); + EXPECT_TRUE(ctx->matches(DocId(3))); + EXPECT_FALSE(ctx->matches(DocId(4))); + EXPECT_TRUE(ctx->matches(DocId(5))); } -TEST_F("cmp(weight) performs GID mapping and forwards to target attribute", WsetValueFixture) { +TEST_F("matches(weight) performs GID mapping and forwards to target attribute", WsetValueFixture) { auto ctx = f.create_context(word_term("foo")); int32_t weight = 0; - EXPECT_FALSE(ctx->cmp(DocId(1), weight)); + EXPECT_FALSE(ctx->matches(DocId(1), weight)); EXPECT_EQUAL(0, weight); // Unchanged - EXPECT_TRUE(ctx->cmp(DocId(2), weight)); + EXPECT_TRUE(ctx->matches(DocId(2), weight)); EXPECT_EQUAL(-5, weight); - EXPECT_TRUE(ctx->cmp(DocId(6), weight)); + EXPECT_TRUE(ctx->matches(DocId(6), weight)); EXPECT_EQUAL(42, weight); } diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp index cbc86b02ada..77504c4ab3c 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/attribute/singlenumericattribute.h> #include <vespa/searchlib/attribute/singlestringattribute.h> #include <vespa/searchlib/attribute/multistringattribute.h> +#include <vespa/searchlib/attribute/elementiterator.h> #include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> @@ -193,12 +194,8 @@ private: // test search iterator functionality - void testStrictSearchIterator(SearchContext & threeHits, - SearchContext & noHits, - const IteratorTester & typeTester); - void testNonStrictSearchIterator(SearchContext & threeHits, - SearchContext & noHits, - const IteratorTester & typeTester); + void testStrictSearchIterator(SearchContext & threeHits, SearchContext & noHits, const IteratorTester & typeTester); + void testNonStrictSearchIterator(SearchContext & threeHits, SearchContext & noHits, const IteratorTester & typeTester); void fillForSearchIteratorTest(IntegerAttribute * ia); void fillForSemiNibbleSearchIteratorTest(IntegerAttribute * ia); void testSearchIterator(); @@ -206,17 +203,20 @@ private: // test search iterator unpacking void fillForSearchIteratorUnpackingTest(IntegerAttribute * ia, bool extra); - void testSearchIteratorUnpacking(const AttributePtr & ptr, - SearchContext & sc, - bool extra, - bool strict); + void testSearchIteratorUnpacking(const AttributePtr & ptr, SearchContext & sc, bool extra, bool strict) { + sc.fetchPostings(strict); + for (bool withElementId : {false, true}) { + testSearchIteratorUnpacking(ptr, sc, extra, strict, withElementId); + } + } + void testSearchIteratorUnpacking(const AttributePtr & ptr, SearchContext & sc, + bool extra, bool strict, bool withElementId); void testSearchIteratorUnpacking(); // test range search template <typename VectorType> - void performRangeSearch(const VectorType & vec, const vespalib::string & term, - const DocSet & expected); + void performRangeSearch(const VectorType & vec, const vespalib::string & term, const DocSet & expected); template <typename VectorType, typename ValueType> void testRangeSearch(const AttributePtr & ptr, uint32_t numDocs, std::vector<ValueType> values); void testRangeSearch(); @@ -224,8 +224,7 @@ private: // test case insensitive search - void performCaseInsensitiveSearch(const StringAttribute & vec, const vespalib::string & term, - const DocSet & expected); + void performCaseInsensitiveSearch(const StringAttribute & vec, const vespalib::string & term, const DocSet & expected); void testCaseInsensitiveSearch(const AttributePtr & ptr); void testCaseInsensitiveSearch(); void testRegexSearch(const AttributePtr & ptr); @@ -252,25 +251,19 @@ private: void requireThatSearchIsWorkingAfterLoadAndClearDoc(); template <typename VectorType, typename ValueType> - void requireThatSearchIsWorkingAfterUpdates(const vespalib::string & name, - const Config & cfg, - ValueType value1, - ValueType value2); + void requireThatSearchIsWorkingAfterUpdates(const vespalib::string & name, const Config & cfg, + ValueType value1, ValueType value2); void requireThatSearchIsWorkingAfterUpdates(); void requireThatFlagAttributeIsWorkingWhenNewDocsAreAdded(); template <typename VectorType, typename ValueType> - void requireThatInvalidSearchTermGivesZeroHits(const vespalib::string & name, - const Config & cfg, - ValueType value); + void requireThatInvalidSearchTermGivesZeroHits(const vespalib::string & name, const Config & cfg, ValueType value); void requireThatInvalidSearchTermGivesZeroHits(); void requireThatFlagAttributeHandlesTheByteRange(); - void requireThatOutOfBoundsSearchTermGivesZeroHits(const vespalib::string &name, - const Config &cfg, - int64_t maxValue); + void requireThatOutOfBoundsSearchTermGivesZeroHits(const vespalib::string &name, const Config &cfg, int64_t maxValue); void requireThatOutOfBoundsSearchTermGivesZeroHits(); // init maps with config objects @@ -620,21 +613,30 @@ void SearchContextTest::testSearch(const ConfigMap & cfgs) { template<typename T, typename A> class Verifier : public search::test::SearchIteratorVerifier { public: - Verifier(T key, const vespalib::string & keyAsString, const vespalib::string & name, const Config & cfg); + Verifier(T key, const vespalib::string & keyAsString, const vespalib::string & name, + const Config & cfg, bool withElementId); ~Verifier(); - SearchIterator::UP create(bool strict) const override { - return _sc->createIterator(&_dummy, strict); + SearchIterator::UP + create(bool strict) const override { + auto search = _sc->createIterator(&_dummy, strict); + if (_withElementId) { + search = std::make_unique<attribute::ElementIterator>(std::move(search), *_sc, _dummy); + } + return search; } private: mutable TermFieldMatchData _dummy; - AttributePtr _attribute; + const bool _withElementId; + AttributePtr _attribute; SearchContextPtr _sc; }; template<typename T, typename A> -Verifier<T, A>::Verifier(T key, const vespalib::string & keyAsString, const vespalib::string & name, const Config & cfg) - :_attribute(AttributeFactory::createAttribute(name + "-initrange", cfg)), - _sc() +Verifier<T, A>::Verifier(T key, const vespalib::string & keyAsString, const vespalib::string & name, + const Config & cfg, bool withElementId) + : _withElementId(withElementId), + _attribute(AttributeFactory::createAttribute(name + "-initrange", cfg)), + _sc() { SearchContextTest::addDocs(*_attribute, getDocIdLimit()); for (uint32_t doc : getExpectedDocIds()) { @@ -648,15 +650,18 @@ Verifier<T, A>::Verifier(T key, const vespalib::string & keyAsString, const vesp } template<typename T, typename A> -Verifier<T, A>::~Verifier() {} +Verifier<T, A>::~Verifier() = default; template<typename T, typename A> void SearchContextTest::testSearchIterator(T key, const vespalib::string &keyAsString, const ConfigMap &cfgs) { - for (const auto & cfg : cfgs) { - Verifier<T, A> verifier(key, keyAsString, cfg.first, cfg.second); - verifier.verify(); + for (bool withElementId : {false, true} ) { + for (const auto & cfg : cfgs) { + Verifier<T, A> verifier(key, keyAsString, cfg.first, cfg.second, withElementId); + verifier.verify(); + } } + } void SearchContextTest::testSearchIteratorConformance() { @@ -935,13 +940,10 @@ SearchContextTest::fillForSearchIteratorUnpackingTest(IntegerAttribute * ia, } void -SearchContextTest::testSearchIteratorUnpacking(const AttributePtr & attr, - SearchContext & sc, - bool extra, - bool strict) +SearchContextTest::testSearchIteratorUnpacking(const AttributePtr & attr, SearchContext & sc, + bool extra, bool strict, bool withElementId) { - LOG(info, - "testSearchIteratorUnpacking: vector '%s'", attr->getName().c_str()); + LOG(info, "testSearchIteratorUnpacking: vector '%s'", attr->getName().c_str()); TermFieldMatchData md; md.reset(100); @@ -950,8 +952,10 @@ SearchContextTest::testSearchIteratorUnpacking(const AttributePtr & attr, pos.setElementWeight(100); md.appendPosition(pos); - sc.fetchPostings(strict); SearchBasePtr sb = sc.createIterator(&md, strict); + if (withElementId) { + sb = std::make_unique<attribute::ElementIterator>(std::move(sb), sc, md); + } sb->initFullRange(); std::vector<int32_t> weights(3); @@ -980,12 +984,30 @@ SearchContextTest::testSearchIteratorUnpacking(const AttributePtr & attr, sb->unpack(2); EXPECT_EQUAL(sb->getDocId(), 2u); EXPECT_EQUAL(md.getDocId(), 2u); - EXPECT_EQUAL(md.getWeight(), weights[1]); + if (withElementId && attr->hasMultiValue() && !attr->hasWeightedSetType()) { + EXPECT_EQUAL(2, md.end()- md.begin()); + EXPECT_EQUAL(md.begin()[0].getElementId(), 0u); + EXPECT_EQUAL(md.begin()[0].getElementWeight(), 1); + EXPECT_EQUAL(md.begin()[1].getElementId(), 1u); + EXPECT_EQUAL(md.begin()[1].getElementWeight(), 1); + } else { + EXPECT_EQUAL(md.getWeight(), weights[1]); + } sb->unpack(3); EXPECT_EQUAL(sb->getDocId(), 3u); EXPECT_EQUAL(md.getDocId(), 3u); - EXPECT_EQUAL(md.getWeight(), weights[2]); + if (withElementId && attr->hasMultiValue() && !attr->hasWeightedSetType()) { + EXPECT_EQUAL(3, md.end()- md.begin()); + EXPECT_EQUAL(md.begin()[0].getElementId(), 0u); + EXPECT_EQUAL(md.begin()[0].getElementWeight(), 1); + EXPECT_EQUAL(md.begin()[1].getElementId(), 1u); + EXPECT_EQUAL(md.begin()[1].getElementWeight(), 1); + EXPECT_EQUAL(md.begin()[2].getElementId(), 2u); + EXPECT_EQUAL(md.begin()[2].getElementWeight(), 1); + } else { + EXPECT_EQUAL(md.getWeight(), weights[2]); + } if (extra) { sb->unpack(4); EXPECT_EQUAL(sb->getDocId(), 4u); @@ -1894,7 +1916,7 @@ SearchContextTest::SearchContextTest() : initStringConfig(); } -SearchContextTest::~SearchContextTest() {} +SearchContextTest::~SearchContextTest() = default; int SearchContextTest::Main() diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index 794e0b2bdf6..58218ecfd65 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -34,6 +34,7 @@ vespa_add_library(searchlib_attribute OBJECT defines.cpp diversity.cpp dociditerator.cpp + elementiterator.cpp enumattribute.cpp enumattributesaver.cpp enumcomparator.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 60b8b1603c8..712288f9a1c 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -65,6 +65,8 @@ using search::queryeval::ParallelWeakAndBlueprint; using search::queryeval::PredicateBlueprint; using search::queryeval::SearchIterator; using search::queryeval::Searchable; +using search::queryeval::SimpleLeafBlueprint; +using search::queryeval::ComplexLeafBlueprint; using search::queryeval::WeightedSetTermBlueprint; using vespalib::geo::ZCurve; using vespalib::string; @@ -77,18 +79,15 @@ namespace { /** * Blueprint for creating regular, stack-based attribute iterators. **/ -class AttributeFieldBlueprint : - public search::queryeval::SimpleLeafBlueprint +class AttributeFieldBlueprint : public SimpleLeafBlueprint { private: ISearchContext::UP _search_context; - AttributeFieldBlueprint(const FieldSpec &field, - const IAttributeVector &attribute, - const string &query_stack, - const attribute::SearchContextParams ¶ms) + AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, + const string &query_stack, const attribute::SearchContextParams ¶ms) : SimpleLeafBlueprint(field), - _search_context(attribute.createSearchContext(QueryTermDecoder::decodeTerm(query_stack), params).release()) + _search_context(attribute.createSearchContext(QueryTermDecoder::decodeTerm(query_stack), params)) { uint32_t estHits = _search_context->approximateHits(); HitEstimate estimate(estHits, estHits == 0); @@ -96,40 +95,30 @@ private: } public: - AttributeFieldBlueprint(const FieldSpec &field, - const IAttributeVector &attribute, - const string &query_stack) - : AttributeFieldBlueprint(field, - attribute, - query_stack, - attribute::SearchContextParams() - .useBitVector(field.isFilter())) - { - } - - AttributeFieldBlueprint(const FieldSpec &field, - const IAttributeVector &attribute, - const IAttributeVector &diversity, - const string &query_stack, - size_t diversityCutoffGroups, - bool diversityCutoffStrict) - : AttributeFieldBlueprint(field, - attribute, - query_stack, + AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, const string &query_stack) + : AttributeFieldBlueprint(field, attribute, query_stack, + attribute::SearchContextParams().useBitVector(field.isFilter())) + {} + + AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, + const IAttributeVector &diversity, const string &query_stack, + size_t diversityCutoffGroups, bool diversityCutoffStrict) + : AttributeFieldBlueprint(field, attribute, query_stack, attribute::SearchContextParams() .diversityAttribute(&diversity) .useBitVector(field.isFilter()) .diversityCutoffGroups(diversityCutoffGroups) .diversityCutoffStrict(diversityCutoffStrict)) - { - } + {} - SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { + SearchIterator::UP + createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { assert(tfmda.size() == 1); return _search_context->createIterator(tfmda[0], strict); } - void fetchPostings(bool strict) override { + void + fetchPostings(bool strict) override { _search_context->fetchPostings(strict); } @@ -139,7 +128,7 @@ public: void AttributeFieldBlueprint::visitMembers(vespalib::ObjectVisitor &visitor) const { - search::queryeval::LeafBlueprint::visitMembers(visitor); + LeafBlueprint::visitMembers(visitor); visit(visitor, "attribute", _search_context->attributeName()); } @@ -149,11 +138,10 @@ template <bool is_strict> struct LocationPreFilterIterator : public OrLikeSearch<is_strict, NoUnpack> { LocationPreFilterIterator(const std::vector<SearchIterator *> &children) : OrLikeSearch<is_strict, NoUnpack>(children, NoUnpack()) {} - virtual void doUnpack(uint32_t) override {} + void doUnpack(uint32_t) override {} }; -class LocationPreFilterBlueprint : - public search::queryeval::ComplexLeafBlueprint +class LocationPreFilterBlueprint : public ComplexLeafBlueprint { private: const IAttributeVector &_attribute; @@ -171,8 +159,8 @@ public: const IAttributeVector &attr(_attribute); for (auto it(rangeVector.begin()), mt(rangeVector.end()); it != mt; it++) { const ZCurve::Range &r(*it); - search::query::Range qr(r.min(), r.max()); - search::query::SimpleRangeTerm rt(qr, "", 0, search::query::Weight(0)); + query::Range qr(r.min(), r.max()); + query::SimpleRangeTerm rt(qr, "", 0, query::Weight(0)); string stack(StackDumpCreator::create(rt)); _rangeSearches.push_back(attr.createSearchContext(QueryTermDecoder::decodeTerm(stack), attribute::SearchContextParams())); @@ -191,22 +179,21 @@ public: bool should_use() const { return _should_use; } - virtual SearchIterator::UP + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { std::vector<SearchIterator *> children; for (auto it(_rangeSearches.begin()), mt(_rangeSearches.end()); it != mt; it++) { - children.push_back((*it)->createIterator(tfmda[0], - strict).release()); + children.push_back((*it)->createIterator(tfmda[0], strict).release()); } if (strict) { - return SearchIterator::UP(new LocationPreFilterIterator<true>(children)); + return std::make_unique<LocationPreFilterIterator<true>>(children); } else { - return SearchIterator::UP(new LocationPreFilterIterator<false>(children)); + return std::make_unique<LocationPreFilterIterator<false>>(children); } } - virtual void fetchPostings(bool strict) override { + void fetchPostings(bool strict) override { for (size_t i(0); i < _rangeSearches.size(); i++) { _rangeSearches[i]->fetchPostings(strict); } @@ -215,12 +202,11 @@ public: //----------------------------------------------------------------------------- -class LocationPostFilterBlueprint : - public search::queryeval::ComplexLeafBlueprint +class LocationPostFilterBlueprint : public ComplexLeafBlueprint { private: const IAttributeVector &_attribute; - search::common::Location _location; + common::Location _location; public: LocationPostFilterBlueprint(const FieldSpec &field, const IAttributeVector &attribute, const Location &loc) @@ -235,46 +221,43 @@ public: setEstimate(estimate); } - const search::common::Location &location() const { return _location; } + const common::Location &location() const { return _location; } - virtual SearchIterator::UP + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &, bool strict) const override { - unsigned int num_docs = _attribute.getNumDocs(); - return SearchIterator::UP(FastS_AllocLocationIterator(num_docs, strict, _location)); + return FastS_AllocLocationIterator(_attribute.getNumDocs(), strict, _location); } }; //----------------------------------------------------------------------------- -Blueprint::UP make_location_blueprint(const FieldSpec &field, const IAttributeVector &attribute, const Location &loc) { - LocationPostFilterBlueprint *post_filter = new LocationPostFilterBlueprint(field, attribute, loc); - Blueprint::UP post_filter_bp(post_filter); - const search::common::Location &location = post_filter->location(); +Blueprint::UP +make_location_blueprint(const FieldSpec &field, const IAttributeVector &attribute, const Location &loc) { + auto post_filter = std::make_unique<LocationPostFilterBlueprint>(field, attribute, loc); + const common::Location &location = post_filter->location(); if (location.getMinX() > location.getMaxX() || location.getMinY() > location.getMaxY()) { - return Blueprint::UP(new queryeval::EmptyBlueprint(field)); + return std::make_unique<queryeval::EmptyBlueprint>(field); } ZCurve::RangeVector rangeVector = ZCurve::find_ranges( location.getMinX(), location.getMinY(), location.getMaxX(), location.getMaxY()); - LocationPreFilterBlueprint *pre_filter = new LocationPreFilterBlueprint(field, attribute, rangeVector); - Blueprint::UP pre_filter_bp(pre_filter); + auto pre_filter = std::make_unique<LocationPreFilterBlueprint>(field, attribute, rangeVector); if (!pre_filter->should_use()) { - return post_filter_bp; + return post_filter; } - AndBlueprint *root = new AndBlueprint(); - Blueprint::UP root_bp(root); - root->addChild(std::move(pre_filter_bp)); - root->addChild(std::move(post_filter_bp)); - return root_bp; + auto root = std::make_unique<AndBlueprint>(); + root->addChild(std::move(pre_filter)); + root->addChild(std::move(post_filter)); + return root; } //----------------------------------------------------------------------------- template <typename SearchType> -class DirectWeightedSetBlueprint : public search::queryeval::ComplexLeafBlueprint +class DirectWeightedSetBlueprint : public ComplexLeafBlueprint { private: HitEstimate _estimate; @@ -283,8 +266,7 @@ private: const IDocumentWeightAttribute &_attr; public: - DirectWeightedSetBlueprint(const FieldSpec &field, - const IDocumentWeightAttribute &attr, size_t size_hint) + DirectWeightedSetBlueprint(const FieldSpec &field, const IDocumentWeightAttribute &attr, size_t size_hint) : ComplexLeafBlueprint(field), _estimate(), _weights(), @@ -315,7 +297,7 @@ public: { assert(tfmda.size() == 1); if (_terms.size() == 0) { - return SearchIterator::UP(new search::queryeval::EmptySearch()); + return std::make_unique<queryeval::EmptySearch>(); } std::vector<DocumentWeightIterator> iterators; const size_t numChildren = _terms.size(); @@ -329,7 +311,7 @@ public: //----------------------------------------------------------------------------- -class DirectWandBlueprint : public search::queryeval::ComplexLeafBlueprint +class DirectWandBlueprint : public queryeval::ComplexLeafBlueprint { private: HitEstimate _estimate; @@ -342,12 +324,8 @@ private: const IDocumentWeightAttribute &_attr; public: - DirectWandBlueprint(const FieldSpec &field, - const IDocumentWeightAttribute &attr, - uint32_t scoresToTrack, - queryeval::wand::score_t scoreThreshold, - double thresholdBoostFactor, - size_t size_hint) + DirectWandBlueprint(const FieldSpec &field, const IDocumentWeightAttribute &attr, uint32_t scoresToTrack, + queryeval::wand::score_t scoreThreshold, double thresholdBoostFactor, size_t size_hint) : ComplexLeafBlueprint(field), _estimate(), _scores(scoresToTrack), @@ -380,20 +358,19 @@ public: SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { assert(tfmda.size() == 1); if (_terms.size() == 0) { - return SearchIterator::UP(new search::queryeval::EmptySearch()); + return std::make_unique<queryeval::EmptySearch>(); } - return search::queryeval::ParallelWeakAndSearch::create(*tfmda[0], - queryeval::ParallelWeakAndSearch::MatchParams(_scores, - _scoreThreshold, - _thresholdBoostFactor, - _scoresAdjustFrequency).setDocIdLimit(get_docid_limit()), + return queryeval::ParallelWeakAndSearch::create(*tfmda[0], + queryeval::ParallelWeakAndSearch::MatchParams(_scores, _scoreThreshold, + _thresholdBoostFactor, _scoresAdjustFrequency) + .setDocIdLimit(get_docid_limit()), _weights, _terms, _attr, strict); } }; //----------------------------------------------------------------------------- -class DirectAttributeBlueprint : public search::queryeval::SimpleLeafBlueprint +class DirectAttributeBlueprint : public queryeval::SimpleLeafBlueprint { private: vespalib::string _attrName; @@ -401,8 +378,7 @@ private: IDocumentWeightAttribute::LookupResult _dict_entry; public: - DirectAttributeBlueprint(const FieldSpec &field, - const vespalib::string & name, + DirectAttributeBlueprint(const FieldSpec &field, const vespalib::string & name, const IDocumentWeightAttribute &attr, const vespalib::string &term) : SimpleLeafBlueprint(field), _attrName(name), @@ -415,13 +391,13 @@ public: SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { assert(tfmda.size() == 1); if (_dict_entry.posting_size == 0) { - return SearchIterator::UP(new search::queryeval::EmptySearch()); + return std::make_unique<queryeval::EmptySearch>(); } - return SearchIterator::UP(new queryeval::DocumentWeightSearchIterator(*tfmda[0], _attr, _dict_entry)); + return std::make_unique<queryeval::DocumentWeightSearchIterator>(*tfmda[0], _attr, _dict_entry); } void visitMembers(vespalib::ObjectVisitor &visitor) const override { - search::queryeval::LeafBlueprint::visitMembers(visitor); + LeafBlueprint::visitMembers(visitor); visit(visitor, "attribute", _attrName); } }; @@ -429,10 +405,7 @@ public: //----------------------------------------------------------------------------- bool check_valid_diversity_attr(const IAttributeVector *attr) { - if (attr == nullptr) { - return false; - } - if (attr->hasMultiValue()) { + if ((attr == nullptr) || attr->hasMultiValue()) { return false; } return (attr->hasEnum() || attr->isIntegerType() || attr->isFloatingPointType()); @@ -452,10 +425,8 @@ private: const IDocumentWeightAttribute *_dwa; public: - CreateBlueprintVisitor(Searchable &searchable, - const IRequestContext &requestContext, - const FieldSpec &field, - const IAttributeVector &attr) + CreateBlueprintVisitor(Searchable &searchable, const IRequestContext &requestContext, + const FieldSpec &field, const IAttributeVector &attr) : CreateBlueprintVisitorHelper(searchable, field, requestContext), _field(field), _attr(attr), @@ -464,11 +435,11 @@ public: template <class TermNode> void visitTerm(TermNode &n, bool simple = false) { if (simple && (_dwa != nullptr) && !_field.isFilter() && n.isRanked()) { - vespalib::string term = search::queryeval::termAsString(n); - setResult(make_UP(new DirectAttributeBlueprint(_field, _attr.getName(), *_dwa, term))); + vespalib::string term = queryeval::termAsString(n); + setResult(std::make_unique<DirectAttributeBlueprint>(_field, _attr.getName(), *_dwa, term)); } else { const string stack = StackDumpCreator::create(n); - setResult(make_UP(new AttributeFieldBlueprint(_field, _attr, stack))); + setResult(std::make_unique<AttributeFieldBlueprint>(_field, _attr, stack)); } } @@ -478,14 +449,12 @@ public: } void visitPredicate(PredicateQuery &query) { - const PredicateAttribute *attr = - dynamic_cast<const PredicateAttribute *>(&_attr); + const PredicateAttribute *attr = dynamic_cast<const PredicateAttribute *>(&_attr); if (!attr) { - LOG(warning, "Trying to apply a PredicateQuery node to a " - "non-predicate attribute."); - setResult(Blueprint::UP(new queryeval::EmptyBlueprint(_field))); + LOG(warning, "Trying to apply a PredicateQuery node to a non-predicate attribute."); + setResult(std::make_unique<queryeval::EmptyBlueprint>(_field)); } else { - setResult(Blueprint::UP(new PredicateBlueprint( _field, *attr, query))); + setResult(std::make_unique<PredicateBlueprint>( _field, *attr, query)); } } @@ -495,31 +464,31 @@ public: void visit(RangeTerm &n) override { const string stack = StackDumpCreator::create(n); - const string term = search::queryeval::termAsString(n); - search::QueryTermSimple parsed_term(term, search::QueryTermSimple::WORD); + const string term = queryeval::termAsString(n); + QueryTermSimple parsed_term(term, QueryTermSimple::WORD); if (parsed_term.getMaxPerGroup() > 0) { const IAttributeVector *diversity(getRequestContext().getAttribute(parsed_term.getDiversityAttribute())); if (check_valid_diversity_attr(diversity)) { - setResult(make_UP(new AttributeFieldBlueprint(_field, _attr, *diversity, stack, - parsed_term.getDiversityCutoffGroups(), - parsed_term.getDiversityCutoffStrict()))); + setResult(std::make_unique<AttributeFieldBlueprint>(_field, _attr, *diversity, stack, + parsed_term.getDiversityCutoffGroups(), + parsed_term.getDiversityCutoffStrict())); } else { - setResult(Blueprint::UP(new queryeval::EmptyBlueprint(_field))); + setResult(std::make_unique<queryeval::EmptyBlueprint>(_field)); } } else { - setResult(make_UP(new AttributeFieldBlueprint(_field, _attr, stack))); + setResult(std::make_unique<AttributeFieldBlueprint>(_field, _attr, stack)); } } void visit(StringTerm & n) override { visitTerm(n, true); } void visit(SubstringTerm & n) override { - search::query::SimpleRegExpTerm re(vespalib::Regexp::make_from_substring(n.getTerm()), - n.getView(), n.getId(), n.getWeight()); + query::SimpleRegExpTerm re(vespalib::Regexp::make_from_substring(n.getTerm()), + n.getView(), n.getId(), n.getWeight()); visitTerm(re); } void visit(SuffixTerm & n) override { - search::query::SimpleRegExpTerm re(vespalib::Regexp::make_from_suffix(n.getTerm()), - n.getView(), n.getId(), n.getWeight()); + query::SimpleRegExpTerm re(vespalib::Regexp::make_from_suffix(n.getTerm()), + n.getView(), n.getId(), n.getWeight()); visitTerm(re); } void visit(PredicateQuery &n) override { visitPredicate(n); } @@ -529,9 +498,9 @@ public: void createDirectWeightedSet(WS *bp, NODE &n) { Blueprint::UP result(bp); for (size_t i = 0; i < n.getChildren().size(); ++i) { - const search::query::Node &node = *n.getChildren()[i]; - vespalib::string term = search::queryeval::termAsString(node); - uint32_t weight = search::queryeval::getWeightFromNode(node).percent(); + const query::Node &node = *n.getChildren()[i]; + vespalib::string term = queryeval::termAsString(node); + uint32_t weight = queryeval::getWeightFromNode(node).percent(); bp->addTerm(term, weight); } setResult(std::move(result)); @@ -541,36 +510,34 @@ public: void createShallowWeightedSet(WS *bp, NODE &n, const FieldSpec &fs) { Blueprint::UP result(bp); for (size_t i = 0; i < n.getChildren().size(); ++i) { - const search::query::Node &node = *n.getChildren()[i]; - uint32_t weight = search::queryeval::getWeightFromNode(node).percent(); + const query::Node &node = *n.getChildren()[i]; + uint32_t weight = queryeval::getWeightFromNode(node).percent(); const string stack = StackDumpCreator::create(node); FieldSpec childfs = bp->getNextChildField(fs); - bp->addTerm(make_UP(new AttributeFieldBlueprint(childfs, _attr, stack)), weight); + bp->addTerm(std::make_unique<AttributeFieldBlueprint>(childfs, _attr, stack), weight); } setResult(std::move(result)); } - void visit(search::query::WeightedSetTerm &n) override { + void visit(query::WeightedSetTerm &n) override { bool isSingleValue = !_attr.hasMultiValue(); bool isString = (_attr.isStringType() && _attr.hasEnum()); bool isInteger = _attr.isIntegerType(); if (isSingleValue && (isString || isInteger)) { - AttributeWeightedSetBlueprint *ws - = new AttributeWeightedSetBlueprint(_field, _attr); - Blueprint::UP result(ws); + auto ws = std::make_unique<AttributeWeightedSetBlueprint>(_field, _attr); for (size_t i = 0; i < n.getChildren().size(); ++i) { - const search::query::Node &node = *n.getChildren()[i]; - uint32_t weight = search::queryeval::getWeightFromNode(node).percent(); - vespalib::string term = search::queryeval::termAsString(node); - search::QueryTermSimple::UP qt; + const query::Node &node = *n.getChildren()[i]; + uint32_t weight = queryeval::getWeightFromNode(node).percent(); + vespalib::string term = queryeval::termAsString(node); + QueryTermSimple::UP qt; if (isInteger) { - qt.reset(new search::QueryTermSimple(term, search::QueryTermSimple::WORD)); + qt = std::make_unique<QueryTermSimple>(term, QueryTermSimple::WORD); } else { - qt.reset(new search::QueryTermBase(term, search::QueryTermSimple::WORD)); + qt = std::make_unique<QueryTermBase>(term, QueryTermSimple::WORD); } ws->addToken(_attr.createSearchContext(std::move(qt), attribute::SearchContextParams()), weight); } - setResult(std::move(result)); + setResult(std::move(ws)); } else { if (_dwa != nullptr) { auto *bp = new DirectWeightedSetBlueprint<queryeval::WeightedSetTermSearch>(_field, *_dwa, n.getChildren().size()); @@ -582,7 +549,7 @@ public: } } - void visit(search::query::DotProduct &n) override { + void visit(query::DotProduct &n) override { if (_dwa != nullptr) { auto *bp = new DirectWeightedSetBlueprint<queryeval::DotProductSearch>(_field, *_dwa, n.getChildren().size()); createDirectWeightedSet(bp, n); @@ -592,7 +559,7 @@ public: } } - void visit(search::query::WandTerm &n) override { + void visit(query::WandTerm &n) override { if (_dwa != nullptr) { auto *bp = new DirectWandBlueprint(_field, *_dwa, n.getTargetNumHits(), n.getScoreThreshold(), n.getThresholdBoostFactor(), @@ -614,8 +581,8 @@ public: Blueprint::UP AttributeBlueprintFactory::createBlueprint(const IRequestContext & requestContext, - const FieldSpec &field, - const search::query::Node &term) + const FieldSpec &field, + const query::Node &term) { const IAttributeVector *attr(requestContext.getAttribute(field.getName())); if (attr == nullptr) { @@ -623,8 +590,7 @@ AttributeBlueprintFactory::createBlueprint(const IRequestContext & requestContex } CreateBlueprintVisitor visitor(*this, requestContext, field, *attr); const_cast<Node &>(term).accept(visitor); - Blueprint::UP bp = visitor.getResult(); - return bp; + return visitor.getResult(); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h index febf7d101a9..e0fa06c5e84 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h @@ -82,7 +82,7 @@ protected: public: AttributeIteratorT(const SC &searchContext, fef::TermFieldMatchData *matchData); - bool seekFast(uint32_t docId) const { return _searchContext.cmp(docId); } + bool seekFast(uint32_t docId) const { return _searchContext.matches(docId); } }; template <typename SC> @@ -100,7 +100,7 @@ protected: public: FilterAttributeIteratorT(const SC &searchContext, fef::TermFieldMatchData *matchData); - bool seekFast(uint32_t docId) const { return _searchContext.cmp(docId); } + bool seekFast(uint32_t docId) const { return _searchContext.matches(docId); } }; diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp index fb47ad0cfcc..9b1e837beac 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp @@ -16,14 +16,14 @@ namespace search { template <typename SC> void AttributeIteratorBase::and_hits_into(const SC & sc, BitVector & result, uint32_t begin_id) const { - result.foreach_truebit([&](uint32_t key) { if ( ! sc.cmp(key)) { result.clearBit(key); }}, begin_id); + result.foreach_truebit([&](uint32_t key) { if ( ! sc.matches(key)) { result.clearBit(key); }}, begin_id); result.invalidateCachedCount(); } template <typename SC> void AttributeIteratorBase::or_hits_into(const SC & sc, BitVector & result, uint32_t begin_id) const { - result.foreach_falsebit([&](uint32_t key) { if ( sc.cmp(key)) { result.setBit(key); }}, begin_id); + result.foreach_falsebit([&](uint32_t key) { if ( sc.matches(key)) { result.setBit(key); }}, begin_id); result.invalidateCachedCount(); } @@ -33,7 +33,7 @@ std::unique_ptr<BitVector> AttributeIteratorBase::get_hits(const SC & sc, uint32_t begin_id) const { BitVector::UP result = BitVector::create(begin_id, getEndId()); for (uint32_t docId(std::max(begin_id, getDocId())); docId < getEndId(); docId++) { - if (sc.cmp(docId)) { + if (sc.matches(docId)) { result->setBit(docId); } } @@ -338,7 +338,7 @@ AttributeIteratorT<SC>::doSeek(uint32_t docId) { if (isAtEnd(docId)) { setAtEnd(); - } else if (_searchContext.cmp(docId, _weight)) { + } else if (_searchContext.matches(docId, _weight)) { setDocId(docId); } } @@ -349,7 +349,7 @@ FilterAttributeIteratorT<SC>::doSeek(uint32_t docId) { if (isAtEnd(docId)) { setAtEnd(); - } else if (_searchContext.cmp(docId)) { + } else if (_searchContext.matches(docId)) { setDocId(docId); } } @@ -359,7 +359,7 @@ void AttributeIteratorStrict<SC>::doSeek(uint32_t docId) { for (uint32_t nextId = docId; !isAtEnd(nextId); ++nextId) { - if (_searchContext.cmp(nextId, _weight)) { + if (_searchContext.matches(nextId, _weight)) { setDocId(nextId); return; } @@ -372,7 +372,7 @@ void FilterAttributeIteratorStrict<SC>::doSeek(uint32_t docId) { for (uint32_t nextId = docId; !isAtEnd(nextId); ++nextId) { - if (_searchContext.cmp(nextId)) { + if (_searchContext.matches(nextId)) { setDocId(nextId); return; } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 5261eb0bb05..4dbcc85d861 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -585,7 +585,7 @@ AttributeVector::createSearchContext(QueryTermSimpleUP term, return getSearch(std::move(term), params); } -AttributeVector::SearchContext::~SearchContext() { } +AttributeVector::SearchContext::~SearchContext() = default; unsigned int AttributeVector::SearchContext::approximateHits() const diff --git a/searchlib/src/vespa/searchlib/attribute/elementiterator.cpp b/searchlib/src/vespa/searchlib/attribute/elementiterator.cpp new file mode 100644 index 00000000000..3be4c478376 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/elementiterator.cpp @@ -0,0 +1,47 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "elementiterator.h" +#include <vespa/searchcommon/attribute/i_search_context.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> + +using search::fef::TermFieldMatchDataPosition; + +namespace search::attribute { + +void +ElementIterator::doSeek(uint32_t docid) { + _search->doSeek(docid); + setDocId(_search->getDocId()); +} + +void +ElementIterator::doUnpack(uint32_t docid) { + _tfmd.reset(docid); + int32_t weight(0); + for (int32_t id = _searchContext.find(docid, 0, weight); id >= 0; id = _searchContext.find(docid, id+1, weight)) { + _tfmd.appendPosition(TermFieldMatchDataPosition(id, 0, weight, 1)); + } +} + +vespalib::Trinary +ElementIterator::is_strict() const { + return _search->is_strict(); +} + +void +ElementIterator::initRange(uint32_t beginid, uint32_t endid) { + SearchIterator::initRange(beginid, endid); + _search->initRange(beginid, endid); + setDocId(_search->getDocId()); +} + +ElementIterator::ElementIterator(SearchIterator::UP search, const ISearchContext & sc, fef::TermFieldMatchData & tfmd) + : _search(std::move(search)), + _searchContext(sc), + _tfmd(tfmd) +{ +} + +ElementIterator::~ElementIterator() = default; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/elementiterator.h b/searchlib/src/vespa/searchlib/attribute/elementiterator.h new file mode 100644 index 00000000000..232139751b6 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/elementiterator.h @@ -0,0 +1,28 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/queryeval/searchiterator.h> + +namespace search::fef { class TermFieldMatchData; } +namespace search::attribute { + +class ISearchContext; + +class ElementIterator : public queryeval::SearchIterator +{ +private: + SearchIterator::UP _search; + const ISearchContext & _searchContext; + fef::TermFieldMatchData & _tfmd; + + void doSeek(uint32_t docid) override; + void doUnpack(uint32_t docid) override; + Trinary is_strict() const override; + void initRange(uint32_t beginid, uint32_t endid) override; +public: + ElementIterator(SearchIterator::UP search, const ISearchContext & sc, fef::TermFieldMatchData & tfmd); + ~ElementIterator(); +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index bfcd7f29f29..c480eec5e88 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -44,8 +44,7 @@ ImportedSearchContext::ImportedSearchContext( } -ImportedSearchContext::~ImportedSearchContext() { -} +ImportedSearchContext::~ImportedSearchContext() = default; unsigned int ImportedSearchContext::approximateHits() const { return _reference_attribute.getNumDocs(); diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index f8b434d4c6c..cfb5371dbb9 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -50,7 +50,7 @@ public: const SearchContextParams& params, const ImportedAttributeVector& imported_attribute, const attribute::IAttributeVector &target_attribute); - ~ImportedSearchContext(); + ~ImportedSearchContext() override; std::unique_ptr<queryeval::SearchIterator> @@ -64,16 +64,16 @@ public: using DocId = IAttributeVector::DocId; - bool cmp(DocId docId, int32_t& weight) const { - return _target_search_context->cmp(getTargetLid(docId), weight); + int32_t find(DocId docId, int32_t elemId, int32_t& weight) const { + return _target_search_context->find(getTargetLid(docId), elemId, weight); } - bool cmp(DocId docId) const { - return _target_search_context->cmp(getTargetLid(docId)); + int32_t find(DocId docId, int32_t elemId) const { + return _target_search_context->find(getTargetLid(docId), elemId); } - bool onCmp(uint32_t docId, int32_t &weight) const override { return cmp(docId, weight); } - bool onCmp(uint32_t docId) const override { return cmp(docId); } + int32_t onFind(uint32_t docId, int32_t elemId, int32_t &weight) const override { return find(docId, elemId, weight); } + int32_t onFind(uint32_t docId, int32_t elemId) const override { return find(docId, elemId); } const ReferenceAttribute& attribute() const noexcept { return _reference_attribute; } diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h index 1e02e4d39b0..9d0cac64adf 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h @@ -71,12 +71,12 @@ public: private: const MultiValueNumericAttribute<B, M> & _toBeSearched; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elemId) const override { + return find(docId, elemId); } bool valid() const override; @@ -86,25 +86,25 @@ public: Int64Range getAsIntegerTerm() const override; - bool cmp(DocId doc, int32_t & weight) const { + int32_t find(DocId doc, int32_t elemId, int32_t & weight) const { MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); - for (const MultiValueType &mv : values) { - if (this->match(mv.value())) { - weight = mv.weight(); - return true; + for (uint32_t i(elemId); i < values.size(); i++) { + if (this->match(values[i].value())) { + weight = values[i].weight(); + return i; } } - return false; + return -1; } - bool cmp(DocId doc) const { + int32_t find(DocId doc, int32_t elemId) const { MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); - for (const MultiValueType &mv : values) { - if (this->match(mv.value())) { - return true; + for (uint32_t i(elemId); i < values.size(); i++) { + if (this->match(values[i].value())) { + return i; } } - return false; + return -1; } std::unique_ptr<queryeval::SearchIterator> @@ -119,12 +119,12 @@ public: private: const MultiValueNumericAttribute<B, M> & _toBeSearched; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elemId) const override { + return find(docId, elemId); } protected: @@ -132,27 +132,27 @@ public: public: ArraySearchContext(std::unique_ptr<QueryTermSimple> qTerm, const NumericAttribute & toBeSearched); - bool cmp(DocId doc, int32_t & weight) const { - uint32_t hitCount = 0; + int32_t find(DocId doc, int32_t elemId, int32_t & weight) const { MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); - for (const MultiValueType &mv : values) { - if (this->match(mv.value())) { - hitCount++; + for (uint32_t i(elemId); i < values.size(); i++) { + if (this->match(values[i].value())) { + weight = 1; + return i; } } - weight = hitCount; + weight = 0; - return hitCount != 0; + return -1; } - bool cmp(DocId doc) const { + int32_t find(DocId doc, int32_t elemId) const { MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); - for (const MultiValueType &mv : values) { - if (this->match(mv.value())) { - return true; + for (uint32_t i(elemId); i < values.size(); i++) { + if (this->match(values[i].value())) { + return i; } } - return false; + return -1; } Int64Range getAsIntegerTerm() const override; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h index 66dbbfeb1da..b25438f8f2a 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h @@ -52,12 +52,12 @@ protected: protected: const MultiValueNumericEnumAttribute<B, M> & _toBeSearched; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elemId) const override { + return find(docId, elemId); } bool valid() const override { return this->isValid(); } @@ -65,31 +65,31 @@ protected: public: SetSearchContext(QueryTermSimpleUP qTerm, const NumericAttribute & toBeSearched); - bool - cmp(DocId doc, int32_t & weight) const + int32_t + find(DocId doc, int32_t elemId, int32_t & weight) const { WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); - for (const WeightedIndex &wi : indices) { - T v = _toBeSearched._enumStore.getValue(wi.value()); + for (uint32_t i(elemId); i < indices.size(); i++) { + T v = _toBeSearched._enumStore.getValue(indices[i].value()); if (this->match(v)) { - weight = wi.weight(); - return true; + weight = indices[i].weight(); + return i; } } - return false; + return -1; } - bool - cmp(DocId doc) const + int32_t + find(DocId doc, int32_t elemId) const { WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); - for (const WeightedIndex &wi : indices) { - T v = _toBeSearched._enumStore.getValue(wi.value()); + for (uint32_t i(elemId); i < indices.size(); i++) { + T v = _toBeSearched._enumStore.getValue(indices[i].value()); if (this->match(v)) { - return true; + return i; } } - return false; + return -1; } Int64Range getAsIntegerTerm() const override; @@ -105,12 +105,12 @@ protected: protected: const MultiValueNumericEnumAttribute<B, M> & _toBeSearched; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elemId) const override { + return find(docId, elemId); } bool valid() const override { return this->isValid(); } @@ -119,34 +119,34 @@ protected: ArraySearchContext(QueryTermSimpleUP qTerm, const NumericAttribute & toBeSearched); Int64Range getAsIntegerTerm() const override; - bool - cmp(DocId doc, int32_t & weight) const + int32_t + find(DocId doc, int32_t elemId, int32_t & weight) const { - uint32_t hitCount = 0; WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); - for (const WeightedIndex &wi : indices) { - T v = _toBeSearched._enumStore.getValue(wi.value()); + for (uint32_t i(elemId); i < indices.size(); i++) { + T v = _toBeSearched._enumStore.getValue(indices[i].value()); if (this->match(v)) { - hitCount++; + weight = 1; + return i; } } - weight = hitCount; + weight = 0; - return hitCount != 0; + return -1; } bool - cmp(DocId doc) const + find(DocId doc, int32_t elemId) const { WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); - for (const WeightedIndex &wi : indices) { - T v = _toBeSearched._enumStore.getValue(wi.value()); + for (uint32_t i(elemId); i < indices.size(); i++) { + T v = _toBeSearched._enumStore.getValue(indices[i].value()); if (this->match(v)) { - return true; + return i; } } - return false; + return -1; } std::unique_ptr<queryeval::SearchIterator> diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h index 7edbcedcb2e..5e5d33419aa 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h @@ -119,10 +119,10 @@ public: const MultiValueStringAttributeT<B, M> & myAttribute() const { return static_cast< const MultiValueStringAttributeT<B, M> & > (attribute()); } - bool onCmp(DocId docId) const override; + int32_t onFind(DocId docId, int32_t elemId) const override; template <typename Collector> - bool collectWeight(DocId doc, int32_t & weight, Collector & collector) const; + int32_t findNextWeight(DocId doc, int32_t elemId, int32_t & weight, Collector & collector) const; }; /* @@ -134,7 +134,7 @@ public: StringImplSearchContext(std::move(qTerm), toBeSearched) { } protected: - bool onCmp(DocId docId, int32_t & weight) const override; + int32_t onFind(DocId docId, int32_t elemId, int32_t &weight) const override; }; /* @@ -146,7 +146,7 @@ public: StringImplSearchContext(std::move(qTerm), toBeSearched) { } protected: - bool onCmp(DocId docId, int32_t & weight) const override; + int32_t onFind(DocId docId, int32_t elemId, int32_t &weight) const override; }; template <typename BT> diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index a68ac218784..858fe579764 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -62,47 +62,47 @@ private: } template <typename B, typename M> -bool -MultiValueStringAttributeT<B, M>::StringSetImplSearchContext::onCmp(DocId doc, int32_t & weight) const +int32_t +MultiValueStringAttributeT<B, M>::StringSetImplSearchContext::onFind(DocId doc, int32_t elemId, int32_t &weight) const { StringAttribute::StringSearchContext::CollectWeight collector; - return this->collectWeight(doc, weight, collector); + return this->findNextWeight(doc, elemId, weight, collector); } template <typename B, typename M> -bool -MultiValueStringAttributeT<B, M>::StringArrayImplSearchContext::onCmp(DocId doc, int32_t & weight) const +int32_t +MultiValueStringAttributeT<B, M>::StringArrayImplSearchContext::onFind(DocId doc, int32_t elemId, int32_t &weight) const { StringAttribute::StringSearchContext::CollectHitCount collector; - return this->collectWeight(doc, weight, collector); + return this->findNextWeight(doc, elemId, weight, collector); } template <typename B, typename M> template <typename Collector> -bool -MultiValueStringAttributeT<B, M>::StringImplSearchContext::collectWeight(DocId doc, int32_t & weight, Collector & collector) const +int32_t +MultiValueStringAttributeT<B, M>::StringImplSearchContext::findNextWeight(DocId doc, int32_t elemId, int32_t & weight, Collector & collector) const { WeightedIndexArrayRef indices(myAttribute()._mvMapping.get(doc)); EnumAccessor<typename B::EnumStore> accessor(myAttribute()._enumStore); - collectMatches(indices, accessor, collector); + int32_t foundElem = findNextMatch(indices, elemId, accessor, collector); weight = collector.getWeight(); - return collector.hasMatch(); + return foundElem; } template <typename B, typename M> -bool -MultiValueStringAttributeT<B, M>::StringImplSearchContext::onCmp(DocId doc) const +int32_t +MultiValueStringAttributeT<B, M>::StringImplSearchContext::onFind(DocId doc, int32_t elemId) const { const MultiValueStringAttributeT<B, M> & attr(static_cast< const MultiValueStringAttributeT<B, M> & > (attribute())); WeightedIndexArrayRef indices(attr._mvMapping.get(doc)); - for (const WeightedIndex &wi : indices) { - if (isMatch(attr._enumStore.getValue(wi.value()))) { - return true; + for (uint32_t i(elemId); i < indices.size(); i++) { + if (isMatch(attr._enumStore.getValue(indices[i].value()))) { + return i; } } - return false; + return -1; } template <typename B, typename M> diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index 48f9d0404c5..7793af8f510 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -113,7 +113,8 @@ template <typename DataT> void PostingListSearchContextT<DataT>::fetchPostings(bool strict) { - assert(!_fetchPostingsDone); + assert (! _fetchPostingsDone); + _fetchPostingsDone = true; if (_uniqueValues < 2u) { return; diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h index 8edbf6cde59..5bdcc3dfe63 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h @@ -41,27 +41,29 @@ private: private: const T * _data; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int elemId) const override { + return find(docId, elemId); } bool valid() const override; public: SingleSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const NumericAttribute & toBeSearched); - bool cmp(DocId docId, int32_t & weight) const { + int32_t find(DocId docId, int32_t elemId, int32_t & weight) const { + if ( elemId != 0) return -1; const T v = _data[docId]; weight = 1; - return this->match(v); + return this->match(v) ? 0 : -1; } - bool cmp(DocId docId) const { + int32_t find(DocId docId, int elemId) const { + if ( elemId != 0) return -1; const T v = _data[docId]; - return this->match(v); + return this->match(v) ? 0 : -1; } Int64Range getAsIntegerTerm() const override; diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h index 7dca10e83e4..096c159115e 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h @@ -58,12 +58,12 @@ protected: protected: const SingleValueNumericEnumAttribute<B> & _toBeSearched; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elemId) const override { + return find(docId, elemId); } bool valid() const override; @@ -72,15 +72,17 @@ protected: Int64Range getAsIntegerTerm() const override; - bool cmp(DocId docId, int32_t & weight) const { + int32_t find(DocId docId, int32_t elemId, int32_t & weight) const { + if ( elemId != 0) return -1; T v = _toBeSearched._enumStore.getValue(_toBeSearched.getEnumIndex(docId)); weight = 1; - return this->match(v); + return this->match(v) ? 0 : -1; } - bool cmp(DocId docId) const { + int32_t find(DocId docId, int32_t elemId) const { + if ( elemId != 0) return -1; T v = _toBeSearched._enumStore.getValue(_toBeSearched.getEnumIndex(docId)); - return this->match(v); + return this->match(v) ? 0 : -1; } std::unique_ptr<queryeval::SearchIterator> diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h index 9320e248160..f5f666bd89f 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h @@ -67,12 +67,12 @@ public: uint32_t _valueShiftMask; uint32_t _wordShift; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elementId, int32_t & weight) const override { + return find(docId, elementId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elementId) const override { + return find(docId, elementId); } bool valid() const override; @@ -80,19 +80,21 @@ public: public: SingleSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const NumericAttribute & toBeSearched); - bool cmp(DocId docId, int32_t & weight) const { + int32_t find(DocId docId, int32_t elemId, int32_t & weight) const { + if ( elemId != 0) return -1; const Word &word = _wordData[docId >> _wordShift]; uint32_t valueShift = (docId & _valueShiftMask) << _valueShiftShift; T v = (word >> valueShift) & _valueMask; weight = 1; - return match(v); + return match(v) ? 0 : -1; } - bool cmp(DocId docId) const { + int32_t find(DocId docId, int32_t elemId) const { + if ( elemId != 0) return -1; const Word &word = _wordData[docId >> _wordShift]; uint32_t valueShift = (docId & _valueShiftMask) << _valueShiftShift; T v = (word >> valueShift) & _valueMask; - return match(v); + return match(v) ? 0 : -1; } Int64Range getAsIntegerTerm() const override; @@ -101,14 +103,10 @@ public: createFilterIterator(fef::TermFieldMatchData * matchData, bool strict) override; }; - SingleValueSmallNumericAttribute(const vespalib::string & baseFileName, - const Config &c, - Word valueMask, - uint32_t valueShiftShift, - uint32_t valueShiftMask, - uint32_t wordShift); + SingleValueSmallNumericAttribute(const vespalib::string & baseFileName, const Config &c, Word valueMask, + uint32_t valueShiftShift, uint32_t valueShiftMask, uint32_t wordShift); - ~SingleValueSmallNumericAttribute(); + ~SingleValueSmallNumericAttribute() override; uint32_t getValueCount(DocId doc) const override { if (doc >= B::getNumDocs()) { @@ -125,7 +123,8 @@ public: bool onLoad() override; void onSave(IAttributeSaveTarget &saveTarget) override; - SearchContext::UP getSearch(std::unique_ptr<QueryTermSimple> term, const attribute::SearchContextParams & params) const override; + SearchContext::UP + getSearch(std::unique_ptr<QueryTermSimple> term, const attribute::SearchContextParams & params) const override; T getFast(DocId doc) const { const Word &word = _wordData[doc >> _wordShift]; diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h index 227c1d0667c..4993b295b37 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h @@ -89,14 +89,15 @@ public: StringSearchContext(std::move(qTerm), toBeSearched) { } protected: - bool onCmp(DocId doc, int32_t & weight) const override { + int32_t onFind(DocId doc, int32_t elemId, int32_t &weight) const override { weight = 1; - return onCmp(doc); + return onFind(doc, elemId); } - bool onCmp(DocId doc) const override { + int32_t onFind(DocId doc, int32_t elemId) const override { + if ( elemId != 0) return -1; const SingleValueStringAttributeT<B> & attr(static_cast<const SingleValueStringAttributeT<B> &>(attribute())); - return isMatch(attr._enumStore.getValue(attr._enumIndices[doc])); + return isMatch(attr._enumStore.getValue(attr._enumIndices[doc])) ? 0 : -1; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index 5ba936b5f52..16a05e5f0a9 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -273,31 +273,31 @@ public: } -bool -StringAttribute::StringSearchContext::onCmp(DocId docId, int32_t & weight) const +int32_t +StringAttribute::StringSearchContext::onFind(DocId docId, int32_t elemId, int32_t &weight) const { WeightedConstChar * buffer = getBuffer(); uint32_t valueCount = attribute().get(docId, buffer, _bufferLen); CollectWeight collector; DirectAccessor accessor; - collectMatches(vespalib::ConstArrayRef<WeightedConstChar>(buffer, std::min(valueCount, _bufferLen)), accessor, collector); + int32_t foundElem = findNextMatch(vespalib::ConstArrayRef<WeightedConstChar>(buffer, std::min(valueCount, _bufferLen)), elemId, accessor, collector); weight = collector.getWeight(); - return collector.hasMatch(); + return foundElem; } -bool -StringAttribute::StringSearchContext::onCmp(DocId docId) const +int32_t +StringAttribute::StringSearchContext::onFind(DocId docId, int32_t elemId) const { WeightedConstChar * buffer = getBuffer(); uint32_t valueCount = attribute().get(docId, buffer, _bufferLen); - for (uint32_t i = 0, m = std::min(valueCount, _bufferLen); (i < m); i++) { + for (uint32_t i = elemId, m = std::min(valueCount, _bufferLen); (i < m); i++) { if (isMatch(buffer[i].getValue())) { - return true; + return i; } } - return false; + return -1; } bool StringAttribute::applyWeight(DocId doc, const FieldValue & fv, const ArithmeticValueUpdate & wAdjust) diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h index 5e0847f3039..c817332af15 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.h +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h @@ -96,7 +96,7 @@ private: class StringSearchContext : public SearchContext { public: StringSearchContext(QueryTermSimpleUP qTerm, const StringAttribute & toBeSearched); - virtual ~StringSearchContext(); + ~StringSearchContext() override; private: bool _isPrefix; bool _isRegex; @@ -147,17 +147,19 @@ private: }; template<typename WeightedT, typename Accessor, typename Collector> - void collectMatches(vespalib::ConstArrayRef<WeightedT> w, const Accessor & ac, Collector & collector) const { - for (const WeightedT &wRef : w) { - if (isMatch(ac.get(wRef.value()))) { - collector.addWeight(wRef.weight()); + int32_t findNextMatch(vespalib::ConstArrayRef<WeightedT> w, int32_t elemId, const Accessor & ac, Collector & collector) const { + for (uint32_t i(elemId); i < w.size(); i++) { + if (isMatch(ac.get(w[i].value()))) { + collector.addWeight(w[i].weight()); + return i; } } + return -1; } - bool onCmp(DocId docId, int32_t & weight) const override; - bool onCmp(DocId docId) const override; + int32_t onFind(DocId docId, int32_t elementId, int32_t &weight) const override; + int32_t onFind(DocId docId, int32_t elementId) const override; bool isPrefix() const { return _isPrefix; } bool isRegex() const { return _isRegex; } @@ -166,7 +168,7 @@ private: const vespalib::Regexp * getRegex() const { return _regex.get(); } private: WeightedConstChar * getBuffer() const { - if (_buffer == NULL) { + if (_buffer == nullptr) { _buffer = new WeightedConstChar[_bufferLen]; } return _buffer; diff --git a/searchlib/src/vespa/searchlib/common/locationiterators.cpp b/searchlib/src/vespa/searchlib/common/locationiterators.cpp index e460729e663..16e465bcd05 100644 --- a/searchlib/src/vespa/searchlib/common/locationiterators.cpp +++ b/searchlib/src/vespa/searchlib/common/locationiterators.cpp @@ -20,7 +20,7 @@ private: public: FastS_2DZLocationIterator(unsigned int numDocs, bool strict, const Location & location); - ~FastS_2DZLocationIterator(); + ~FastS_2DZLocationIterator() override; }; @@ -39,7 +39,7 @@ FastS_2DZLocationIterator(unsigned int numDocs, }; -FastS_2DZLocationIterator::~FastS_2DZLocationIterator() {} +FastS_2DZLocationIterator::~FastS_2DZLocationIterator() = default; void @@ -103,10 +103,8 @@ FastS_2DZLocationIterator::doUnpack(uint32_t docId) } -search::queryeval::SearchIterator * -FastS_AllocLocationIterator(unsigned int numDocs, - bool strict, - const Location & location) +std::unique_ptr<search::queryeval::SearchIterator> +FastS_AllocLocationIterator(unsigned int numDocs, bool strict, const Location & location) { - return new FastS_2DZLocationIterator(numDocs, strict, location); + return std::make_unique<FastS_2DZLocationIterator>(numDocs, strict, location); } diff --git a/searchlib/src/vespa/searchlib/common/locationiterators.h b/searchlib/src/vespa/searchlib/common/locationiterators.h index b913305873f..e345bcae4fe 100644 --- a/searchlib/src/vespa/searchlib/common/locationiterators.h +++ b/searchlib/src/vespa/searchlib/common/locationiterators.h @@ -2,10 +2,10 @@ #pragma once -#include <vespa/searchlib/queryeval/iterators.h> +#include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/common/location.h> -search::queryeval::SearchIterator * +std::unique_ptr<search::queryeval::SearchIterator> FastS_AllocLocationIterator(unsigned int numDocs, bool strict, const search::common::Location & location); diff --git a/searchlib/src/vespa/searchlib/common/sortresults.cpp b/searchlib/src/vespa/searchlib/common/sortresults.cpp index ed86014f7b3..e39f11f56b2 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.cpp +++ b/searchlib/src/vespa/searchlib/common/sortresults.cpp @@ -459,16 +459,16 @@ public: default: case 4: r |= _data[a._idx + a._pos + 3] << 0; - //@fallthrough@ + [[fallthrough]]; case 3: r |= _data[a._idx + a._pos + 2] << 8; - //@fallthrough@ + [[fallthrough]]; case 2: r |= _data[a._idx + a._pos + 1] << 16; - //@fallthrough@ + [[fallthrough]]; case 1: r |= _data[a._idx + a._pos + 0] << 24; - //@fallthrough@ + [[fallthrough]]; case 0:; } a._pos += std::min(4u, left); diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_hash.h b/searchlib/src/vespa/searchlib/predicate/predicate_hash.h index 938b1bc5542..861a94d1990 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_hash.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_hash.h @@ -75,30 +75,30 @@ struct PredicateHash { // handle the last 23 bytes c += origLen; switch(len) { // all the case statements fall through - case 23: c+=((0xffLL & aKey[offset+22])<<56); //@fallthrough@ - case 22: c+=((0xffLL & aKey[offset+21])<<48); //@fallthrough@ - case 21: c+=((0xffLL & aKey[offset+20])<<40); //@fallthrough@ - case 20: c+=((0xffLL & aKey[offset+19])<<32); //@fallthrough@ - case 19: c+=((0xffLL & aKey[offset+18])<<24); //@fallthrough@ - case 18: c+=((0xffLL & aKey[offset+17])<<16); //@fallthrough@ - case 17: c+=((0xffLL & aKey[offset+16])<<8); //@fallthrough@ + case 23: c+=((0xffLL & aKey[offset+22])<<56); [[fallthrough]]; + case 22: c+=((0xffLL & aKey[offset+21])<<48); [[fallthrough]]; + case 21: c+=((0xffLL & aKey[offset+20])<<40); [[fallthrough]]; + case 20: c+=((0xffLL & aKey[offset+19])<<32); [[fallthrough]]; + case 19: c+=((0xffLL & aKey[offset+18])<<24); [[fallthrough]]; + case 18: c+=((0xffLL & aKey[offset+17])<<16); [[fallthrough]]; + case 17: c+=((0xffLL & aKey[offset+16])<<8); [[fallthrough]]; // the first byte of c is reserved for the length - case 16: b+=((0xffLL & aKey[offset+15])<<56); //@fallthrough@ - case 15: b+=((0xffLL & aKey[offset+14])<<48); //@fallthrough@ - case 14: b+=((0xffLL & aKey[offset+13])<<40); //@fallthrough@ - case 13: b+=((0xffLL & aKey[offset+12])<<32); //@fallthrough@ - case 12: b+=((0xffLL & aKey[offset+11])<<24); //@fallthrough@ - case 11: b+=((0xffLL & aKey[offset+10])<<16); //@fallthrough@ - case 10: b+=((0xffLL & aKey[offset+ 9])<<8); //@fallthrough@ - case 9: b+=( 0xffLL & aKey[offset+ 8]); //@fallthrough@ - case 8: a+=((0xffLL & aKey[offset+ 7])<<56); //@fallthrough@ - case 7: a+=((0xffLL & aKey[offset+ 6])<<48); //@fallthrough@ - case 6: a+=((0xffLL & aKey[offset+ 5])<<40); //@fallthrough@ - case 5: a+=((0xffLL & aKey[offset+ 4])<<32); //@fallthrough@ - case 4: a+=((0xffLL & aKey[offset+ 3])<<24); //@fallthrough@ - case 3: a+=((0xffLL & aKey[offset+ 2])<<16); //@fallthrough@ - case 2: a+=((0xffLL & aKey[offset+ 1])<<8); //@fallthrough@ - case 1: a+=( 0xffLL & aKey[offset+ 0]); //@fallthrough@ + case 16: b+=((0xffLL & aKey[offset+15])<<56); [[fallthrough]]; + case 15: b+=((0xffLL & aKey[offset+14])<<48); [[fallthrough]]; + case 14: b+=((0xffLL & aKey[offset+13])<<40); [[fallthrough]]; + case 13: b+=((0xffLL & aKey[offset+12])<<32); [[fallthrough]]; + case 12: b+=((0xffLL & aKey[offset+11])<<24); [[fallthrough]]; + case 11: b+=((0xffLL & aKey[offset+10])<<16); [[fallthrough]]; + case 10: b+=((0xffLL & aKey[offset+ 9])<<8); [[fallthrough]]; + case 9: b+=( 0xffLL & aKey[offset+ 8]); [[fallthrough]]; + case 8: a+=((0xffLL & aKey[offset+ 7])<<56); [[fallthrough]]; + case 7: a+=((0xffLL & aKey[offset+ 6])<<48); [[fallthrough]]; + case 6: a+=((0xffLL & aKey[offset+ 5])<<40); [[fallthrough]]; + case 5: a+=((0xffLL & aKey[offset+ 4])<<32); [[fallthrough]]; + case 4: a+=((0xffLL & aKey[offset+ 3])<<24); [[fallthrough]]; + case 3: a+=((0xffLL & aKey[offset+ 2])<<16); [[fallthrough]]; + case 2: a+=((0xffLL & aKey[offset+ 1])<<8); [[fallthrough]]; + case 1: a+=( 0xffLL & aKey[offset+ 0]); // case 0: nothing left to add } diff --git a/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h b/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h index ea051aa9318..7cf323aa106 100644 --- a/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h +++ b/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h @@ -5,8 +5,7 @@ #include "searchiterator.h" #include <vespa/searchlib/fef/termfieldmatchdataarray.h> -namespace search { -namespace queryeval { +namespace search::queryeval { /** * A term iterator wrapper used to hide detailed match @@ -22,16 +21,14 @@ private: SearchIterator::UP _search; fef::TermFieldMatchData *_tfmdp; - BooleanMatchIteratorWrapper(const BooleanMatchIteratorWrapper &); - BooleanMatchIteratorWrapper &operator=(const BooleanMatchIteratorWrapper &); - protected: void doSeek(uint32_t docid) override; void doUnpack(uint32_t docid) override; Trinary is_strict() const override { return _search->is_strict(); } void initRange(uint32_t beginid, uint32_t endid) override { + SearchIterator::initRange(beginid, endid); _search->initRange(beginid, endid); - SearchIterator::initRange(_search->getDocId()+1, _search->getEndId()); + setDocId(_search->getDocId()); } public: @@ -49,12 +46,9 @@ public: * @param search internal search, must be a term iterator * @param match term match data used by the internal iterator **/ - BooleanMatchIteratorWrapper(SearchIterator::UP search, - const fef::TermFieldMatchDataArray &matchData); + BooleanMatchIteratorWrapper(SearchIterator::UP search, const fef::TermFieldMatchDataArray &matchData); void visitMembers(vespalib::ObjectVisitor &visitor) const override; }; -} // namespace queryeval -} // namespace search - +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ApplicationInstanceGenerator.java new file mode 100644 index 00000000000..ec2702bcfaf --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ApplicationInstanceGenerator.java @@ -0,0 +1,13 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.application; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; + +/** + * @author hakon + */ +public interface ApplicationInstanceGenerator { + /** Make an ApplicationInstance based on current service status. */ + ApplicationInstance makeApplicationInstance(ServiceStatusProvider serviceStatusProvider); +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerAppGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerAppGenerator.java new file mode 100644 index 00000000000..76ca59cf583 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerAppGenerator.java @@ -0,0 +1,67 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.application; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Class for generating an ApplicationInstance for the synthesized config server application. + * + * @author hakon + */ +public class ConfigServerAppGenerator implements ApplicationInstanceGenerator { + private final List<String> hostnames; + + public ConfigServerAppGenerator(List<String> hostnames) { + this.hostnames = hostnames; + } + + @Override + public ApplicationInstance makeApplicationInstance(ServiceStatusProvider statusProvider) { + Set<ServiceInstance> serviceInstances = hostnames.stream() + .map(hostname -> makeServiceInstance(hostname, statusProvider)) + .collect(Collectors.toSet()); + + ServiceCluster serviceCluster = new ServiceCluster( + ConfigServerApplication.CLUSTER_ID, + ConfigServerApplication.SERVICE_TYPE, + serviceInstances); + + Set<ServiceCluster> serviceClusters = new HashSet<>(); + serviceClusters.add(serviceCluster); + + ApplicationInstance applicationInstance = new ApplicationInstance( + ConfigServerApplication.TENANT_ID, + ConfigServerApplication.APPLICATION_INSTANCE_ID, + serviceClusters); + + // Fill back-references + serviceCluster.setApplicationInstance(applicationInstance); + for (ServiceInstance serviceInstance : serviceCluster.serviceInstances()) { + serviceInstance.setServiceCluster(serviceCluster); + } + + return applicationInstance; + } + + private ServiceInstance makeServiceInstance(String hostname, ServiceStatusProvider statusProvider) { + ConfigId configId = new ConfigId(ConfigServerApplication.CONFIG_ID_PREFIX + hostname); + ServiceStatus status = statusProvider.getStatus( + ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), + ConfigServerApplication.CLUSTER_ID, + ConfigServerApplication.SERVICE_TYPE, + configId); + + return new ServiceInstance(configId, new HostName(hostname), status); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplication.java index 120a12609e1..132bb0927b8 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplication.java @@ -3,22 +3,11 @@ package com.yahoo.vespa.service.monitor.application; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ClusterId; -import com.yahoo.vespa.applicationmodel.ConfigId; -import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceCluster; -import com.yahoo.vespa.applicationmodel.ServiceInstance; -import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * A service/application model of the config server with health status. */ @@ -36,34 +25,4 @@ public class ConfigServerApplication extends HostedVespaApplication { super("zone-config-servers", NodeType.config, ClusterSpec.Type.admin, ClusterSpec.Id.from("zone-config-servers")); } - - public ApplicationInstance toApplicationInstance(List<String> hostnames) { - Set<ServiceInstance> serviceInstances = hostnames.stream() - .map(hostname -> new ServiceInstance( - new ConfigId(CONFIG_ID_PREFIX + hostname), - new HostName(hostname), - ServiceStatus.NOT_CHECKED)) - .collect(Collectors.toSet()); - - ServiceCluster serviceCluster = new ServiceCluster( - CLUSTER_ID, - SERVICE_TYPE, - serviceInstances); - - Set<ServiceCluster> serviceClusters = - Stream.of(serviceCluster).collect(Collectors.toSet()); - - ApplicationInstance applicationInstance = new ApplicationInstance( - TENANT_ID, - APPLICATION_INSTANCE_ID, - serviceClusters); - - // Fill back-references - serviceCluster.setApplicationInstance(applicationInstance); - for (ServiceInstance serviceInstance : serviceCluster.serviceInstances()) { - serviceInstance.setServiceCluster(serviceCluster); - } - - return applicationInstance; - } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/DeployedAppGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/DeployedAppGenerator.java new file mode 100644 index 00000000000..2691a8bf1ee --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/DeployedAppGenerator.java @@ -0,0 +1,127 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.application; + +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceClusterKey; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Class to generate an ApplicationInstance given service status for a standard (deployed) application. + * + * @author hakon + */ +public class DeployedAppGenerator implements ApplicationInstanceGenerator { + public static final String CLUSTER_ID_PROPERTY_NAME = "clustername"; + + private final ApplicationInfo applicationInfo; + private final Zone zone; + + public DeployedAppGenerator(ApplicationInfo applicationInfo, Zone zone) { + this.applicationInfo = applicationInfo; + this.zone = zone; + } + + @Override + public ApplicationInstance makeApplicationInstance(ServiceStatusProvider serviceStatusProvider) { + Map<ServiceClusterKey, Set<ServiceInstance>> groupedServiceInstances = new HashMap<>(); + + for (HostInfo host : applicationInfo.getModel().getHosts()) { + HostName hostName = new HostName(host.getHostname()); + for (ServiceInfo serviceInfo : host.getServices()) { + ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo); + ServiceInstance serviceInstance = + toServiceInstance( + applicationInfo.getApplicationId(), + serviceClusterKey.clusterId(), + serviceInfo, + hostName, + serviceStatusProvider); + + if (!groupedServiceInstances.containsKey(serviceClusterKey)) { + groupedServiceInstances.put(serviceClusterKey, new HashSet<>()); + } + groupedServiceInstances.get(serviceClusterKey).add(serviceInstance); + } + } + + Set<ServiceCluster> serviceClusters = groupedServiceInstances.entrySet().stream() + .map(entry -> new ServiceCluster( + entry.getKey().clusterId(), + entry.getKey().serviceType(), + entry.getValue())) + .collect(Collectors.toSet()); + + ApplicationInstance applicationInstance = new ApplicationInstance( + new TenantId(applicationInfo.getApplicationId().tenant().toString()), + toApplicationInstanceId(applicationInfo, zone), + serviceClusters); + + // Fill back-references + for (ServiceCluster serviceCluster : applicationInstance.serviceClusters()) { + serviceCluster.setApplicationInstance(applicationInstance); + for (ServiceInstance serviceInstance : serviceCluster.serviceInstances()) { + serviceInstance.setServiceCluster(serviceCluster); + } + } + + return applicationInstance; + } + + static ClusterId getClusterId(ServiceInfo serviceInfo) { + return new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse("")); + } + + private ServiceClusterKey toServiceClusterKey(ServiceInfo serviceInfo) { + ClusterId clusterId = getClusterId(serviceInfo); + ServiceType serviceType = toServiceType(serviceInfo); + return new ServiceClusterKey(clusterId, serviceType); + } + + private ServiceInstance toServiceInstance( + ApplicationId applicationId, + ClusterId clusterId, + ServiceInfo serviceInfo, + HostName hostName, + ServiceStatusProvider serviceStatusProvider) { + ConfigId configId = new ConfigId(serviceInfo.getConfigId()); + + ServiceStatus status = serviceStatusProvider.getStatus( + applicationId, + clusterId, + toServiceType(serviceInfo), configId); + + return new ServiceInstance(configId, hostName, status); + } + + private ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) { + return new ApplicationInstanceId(String.format("%s:%s:%s:%s", + applicationInfo.getApplicationId().application().value(), + zone.environment().value(), + zone.region().value(), + applicationInfo.getApplicationId().instance().value())); + } + + private ServiceType toServiceType(ServiceInfo serviceInfo) { + return new ServiceType(serviceInfo.getServiceType()); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ModelGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ModelGenerator.java index c4952979518..9da449289a7 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ModelGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ModelGenerator.java @@ -1,33 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.monitor.internal; -import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModel; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; -import com.yahoo.vespa.applicationmodel.ClusterId; -import com.yahoo.vespa.applicationmodel.ConfigId; -import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceCluster; -import com.yahoo.vespa.applicationmodel.ServiceClusterKey; -import com.yahoo.vespa.applicationmodel.ServiceInstance; -import com.yahoo.vespa.applicationmodel.ServiceStatus; -import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; -import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; +import com.yahoo.vespa.service.monitor.application.ApplicationInstanceGenerator; +import com.yahoo.vespa.service.monitor.application.ConfigServerAppGenerator; +import com.yahoo.vespa.service.monitor.application.DeployedAppGenerator; -import java.util.HashMap; -import java.util.HashSet; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -36,6 +24,16 @@ import java.util.stream.Collectors; public class ModelGenerator { public static final String CLUSTER_ID_PROPERTY_NAME = "clustername"; + private final List<ApplicationInstanceGenerator> staticGenerators; + + public ModelGenerator(List<String> configServerHosts) { + if (configServerHosts.isEmpty()) { + staticGenerators = Collections.emptyList(); + } else { + staticGenerators = Collections.singletonList(new ConfigServerAppGenerator(configServerHosts)); + } + } + /** * Create service model based primarily on super model. * @@ -44,113 +42,15 @@ public class ModelGenerator { ServiceModel toServiceModel( SuperModel superModel, Zone zone, - List<String> configServerHosts, ServiceStatusProvider serviceStatusProvider) { - Map<ApplicationInstanceReference, ApplicationInstance> applicationInstances = new HashMap<>(); + List<ApplicationInstanceGenerator> generators = new ArrayList<>(staticGenerators); + superModel.getAllApplicationInfos() + .forEach(info -> generators.add(new DeployedAppGenerator(info, zone))); - for (ApplicationInfo applicationInfo : superModel.getAllApplicationInfos()) { - - ApplicationInstance applicationInstance = toApplicationInstance( - applicationInfo, - zone, - serviceStatusProvider); - applicationInstances.put(applicationInstance.reference(), applicationInstance); - } - - // The config server is part of the service model (but not super model) - if (!configServerHosts.isEmpty()) { - ConfigServerApplication configServerApplication = ConfigServerApplication.CONFIG_SERVER_APPLICATION; - ApplicationInstance configServerApplicationInstance = - configServerApplication.toApplicationInstance(configServerHosts); - applicationInstances.put(configServerApplicationInstance.reference(), configServerApplicationInstance); - } + Map<ApplicationInstanceReference, ApplicationInstance> applicationInstances = generators.stream() + .map(generator -> generator.makeApplicationInstance(serviceStatusProvider)) + .collect(Collectors.toMap(ApplicationInstance::reference, Function.identity())); return new ServiceModel(applicationInstances); } - - ApplicationInstance toApplicationInstance( - ApplicationInfo applicationInfo, - Zone zone, - ServiceStatusProvider serviceStatusProvider) { - Map<ServiceClusterKey, Set<ServiceInstance>> groupedServiceInstances = new HashMap<>(); - - for (HostInfo host : applicationInfo.getModel().getHosts()) { - HostName hostName = new HostName(host.getHostname()); - for (ServiceInfo serviceInfo : host.getServices()) { - ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo); - ServiceInstance serviceInstance = - toServiceInstance( - applicationInfo.getApplicationId(), - serviceClusterKey.clusterId(), - serviceInfo, - hostName, - serviceStatusProvider); - - if (!groupedServiceInstances.containsKey(serviceClusterKey)) { - groupedServiceInstances.put(serviceClusterKey, new HashSet<>()); - } - groupedServiceInstances.get(serviceClusterKey).add(serviceInstance); - } - } - - Set<ServiceCluster> serviceClusters = groupedServiceInstances.entrySet().stream() - .map(entry -> new ServiceCluster( - entry.getKey().clusterId(), - entry.getKey().serviceType(), - entry.getValue())) - .collect(Collectors.toSet()); - - ApplicationInstance applicationInstance = new ApplicationInstance( - new TenantId(applicationInfo.getApplicationId().tenant().toString()), - toApplicationInstanceId(applicationInfo, zone), - serviceClusters); - - // Fill back-references - for (ServiceCluster serviceCluster : applicationInstance.serviceClusters()) { - serviceCluster.setApplicationInstance(applicationInstance); - for (ServiceInstance serviceInstance : serviceCluster.serviceInstances()) { - serviceInstance.setServiceCluster(serviceCluster); - } - } - - return applicationInstance; - } - - static ClusterId getClusterId(ServiceInfo serviceInfo) { - return new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse("")); - } - - private ServiceClusterKey toServiceClusterKey(ServiceInfo serviceInfo) { - ClusterId clusterId = getClusterId(serviceInfo); - ServiceType serviceType = toServiceType(serviceInfo); - return new ServiceClusterKey(clusterId, serviceType); - } - - private ServiceInstance toServiceInstance( - ApplicationId applicationId, - ClusterId clusterId, - ServiceInfo serviceInfo, - HostName hostName, - ServiceStatusProvider serviceStatusProvider) { - ConfigId configId = new ConfigId(serviceInfo.getConfigId()); - - ServiceStatus status = serviceStatusProvider.getStatus( - applicationId, - clusterId, - toServiceType(serviceInfo), configId); - - return new ServiceInstance(configId, hostName, status); - } - - private ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) { - return new ApplicationInstanceId(String.format("%s:%s:%s:%s", - applicationInfo.getApplicationId().application().value(), - zone.environment().value(), - zone.region().value(), - applicationInfo.getApplicationId().instance().value())); - } - - private ServiceType toServiceType(ServiceInfo serviceInfo) { - return new ServiceType(serviceInfo.getServiceType()); - } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorImpl.java index b2b6538fe6c..97c4fdda0f3 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorImpl.java @@ -11,6 +11,8 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; +import com.yahoo.vespa.service.monitor.internal.health.HealthMonitorManager; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl; import java.util.Collections; import java.util.List; @@ -28,7 +30,6 @@ public class ServiceMonitorImpl implements ServiceMonitor { Metric metric, Timer timer) { Zone zone = superModelProvider.getZone(); - List<String> configServerHosts = toConfigServerList(configserverConfig); ServiceMonitorMetrics metrics = new ServiceMonitorMetrics(metric, timer); UnionMonitorManager monitorManager = new UnionMonitorManager( @@ -39,9 +40,8 @@ public class ServiceMonitorImpl implements ServiceMonitor { SuperModelListenerImpl superModelListener = new SuperModelListenerImpl( monitorManager, metrics, - new ModelGenerator(), - zone, - configServerHosts); + new ModelGenerator(toConfigServerList(configserverConfig)), + zone); superModelListener.start(superModelProvider); serviceModelCache = new ServiceModelCache(superModelListener, timer); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImpl.java index 5e309d3c18d..b2f3617131b 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImpl.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.service.monitor.ServiceModel; -import java.util.List; import java.util.function.Supplier; import java.util.logging.Logger; @@ -19,24 +18,21 @@ public class SuperModelListenerImpl implements SuperModelListener, Supplier<Serv private final ServiceMonitorMetrics metrics; private final ModelGenerator modelGenerator; private final Zone zone; - private final List<String> configServerHosts; - // superModel and slobrokMonitorManager are always updated together + // superModel and monitorManager are always updated together // and atomically using this monitor. private final Object monitor = new Object(); - private final MonitorManager slobrokMonitorManager; + private final MonitorManager monitorManager; private SuperModel superModel; - SuperModelListenerImpl(MonitorManager slobrokMonitorManager, + SuperModelListenerImpl(MonitorManager monitorManager, ServiceMonitorMetrics metrics, ModelGenerator modelGenerator, - Zone zone, - List<String> configServerHosts) { - this.slobrokMonitorManager = slobrokMonitorManager; + Zone zone) { + this.monitorManager = monitorManager; this.metrics = metrics; this.modelGenerator = modelGenerator; this.zone = zone; - this.configServerHosts = configServerHosts; } void start(SuperModelProvider superModelProvider) { @@ -46,7 +42,7 @@ public class SuperModelListenerImpl implements SuperModelListener, Supplier<Serv // asynchronously even before snapshot() returns. this.superModel = superModelProvider.snapshot(this); superModel.getAllApplicationInfos().stream().forEach(application -> - slobrokMonitorManager.applicationActivated(superModel, application)); + monitorManager.applicationActivated(superModel, application)); } } @@ -54,7 +50,7 @@ public class SuperModelListenerImpl implements SuperModelListener, Supplier<Serv public void applicationActivated(SuperModel superModel, ApplicationInfo application) { synchronized (monitor) { this.superModel = superModel; - slobrokMonitorManager.applicationActivated(superModel, application); + monitorManager.applicationActivated(superModel, application); } } @@ -62,7 +58,7 @@ public class SuperModelListenerImpl implements SuperModelListener, Supplier<Serv public void applicationRemoved(SuperModel superModel, ApplicationId id) { synchronized (monitor) { this.superModel = superModel; - slobrokMonitorManager.applicationRemoved(superModel, id); + monitorManager.applicationRemoved(superModel, id); } } @@ -75,11 +71,7 @@ public class SuperModelListenerImpl implements SuperModelListener, Supplier<Serv dummy(measurement); // WARNING: The slobrok monitor manager may be out-of-sync with super model (no locking) - return modelGenerator.toServiceModel( - superModel, - zone, - configServerHosts, - slobrokMonitorManager); + return modelGenerator.toServiceModel(superModel, zone, monitorManager); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManager.java index e224d6bfd12..82d2043bd17 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManager.java @@ -9,7 +9,10 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; import com.yahoo.vespa.service.monitor.application.ZoneApplication; +import com.yahoo.vespa.service.monitor.internal.health.HealthMonitorManager; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl; /** * @author hakon @@ -32,6 +35,12 @@ public class UnionMonitorManager implements MonitorManager { ClusterId clusterId, ServiceType serviceType, ConfigId configId) { + + if (applicationId.equals(ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId())) { + // todo: use health + return ServiceStatus.NOT_CHECKED; + } + MonitorManager monitorManager = useHealth(applicationId, clusterId, serviceType) ? healthMonitorManager : slobrokMonitorManager; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java index 072886098d7..5a4b7251ae2 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.internal; +package com.yahoo.vespa.service.monitor.internal.health; import com.google.inject.Inject; import com.yahoo.config.model.api.ApplicationInfo; @@ -10,6 +10,7 @@ import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.service.monitor.application.ZoneApplication; +import com.yahoo.vespa.service.monitor.internal.MonitorManager; /** * @author hakon @@ -27,7 +28,10 @@ public class HealthMonitorManager implements MonitorManager { } @Override - public ServiceStatus getStatus(ApplicationId applicationId, ClusterId clusterId, ServiceType serviceType, ConfigId configId) { + public ServiceStatus getStatus(ApplicationId applicationId, + ClusterId clusterId, + ServiceType serviceType, + ConfigId configId) { // TODO: Do proper health check if (ZoneApplication.isNodeAdminService(applicationId, clusterId, serviceType)) { return ServiceStatus.UP; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitor.java index e0195e11759..a857be84cc7 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitor.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.internal; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.slobrok; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorManagerImpl.java index 81173f9d835..aaaab22e742 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorManagerImpl.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.internal; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.slobrok; import com.google.inject.Inject; import com.yahoo.config.model.api.ApplicationInfo; @@ -13,6 +13,7 @@ import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.service.monitor.SlobrokApi; +import com.yahoo.vespa.service.monitor.internal.MonitorManager; import java.util.HashMap; import java.util.List; @@ -75,7 +76,8 @@ public class SlobrokMonitorManagerImpl implements SuperModelListener, SlobrokApi @Override public ServiceStatus getStatus(ApplicationId applicationId, - ClusterId clusterId, ServiceType serviceType, + ClusterId clusterId, + ServiceType serviceType, ConfigId configId) { Optional<String> slobrokServiceName = findSlobrokServiceName(serviceType, configId); if (slobrokServiceName.isPresent()) { diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplicationTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/application/ConfigServerAppGeneratorTest.java index 7fa6f82e183..58f99786017 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplicationTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/application/ConfigServerAppGeneratorTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.service.monitor.application; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import org.junit.Test; import java.util.List; @@ -11,8 +12,11 @@ import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; -public class ConfigServerApplicationTest { +public class ConfigServerAppGeneratorTest { private static final String configServer1 = "cfg1.yahoo.com"; private static final String configServer2 = "cfg2.yahoo.com"; private static final String configServer3 = "cfg3.yahoo.com"; @@ -21,11 +25,13 @@ public class ConfigServerApplicationTest { configServer2, configServer3).collect(Collectors.toList()); + private final ServiceStatusProvider statusProvider = mock(ServiceStatusProvider.class); + @Test public void toApplicationInstance() throws Exception { - ConfigServerApplication application = ConfigServerApplication.CONFIG_SERVER_APPLICATION; - ApplicationInstance applicationInstance = - application.toApplicationInstance(configServerList); + when(statusProvider.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.NOT_CHECKED); + ApplicationInstance applicationInstance = new ConfigServerAppGenerator(configServerList) + .makeApplicationInstance(statusProvider); assertEquals( ConfigServerApplication.APPLICATION_INSTANCE_ID, diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ExampleModel.java index fca1512e3ea..186b22cf4ec 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ExampleModel.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ExampleModel.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitor; import java.util.ArrayList; import java.util.Arrays; @@ -51,13 +52,13 @@ public class ExampleModel { return new SuperModel(applicationInfos); } - static ApplicationBuilder createApplication(String tenant, - String applicationName) { + public static ApplicationBuilder createApplication(String tenant, + String applicationName) { return new ApplicationBuilder(tenant, applicationName); } - static class ApplicationBuilder { + public static class ApplicationBuilder { private final String tenant; private final String applicationName; private final List<ClusterBuilder> clusters = new ArrayList<>(); @@ -80,7 +81,7 @@ public class ExampleModel { hosts); } - ApplicationInfo build() { + public ApplicationInfo build() { List<String> allHosts = clusters.stream() .flatMap(clusterBuilder -> clusterBuilder.hosts.stream()) .distinct() diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ModelGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ModelGeneratorTest.java index 6e9fc25382a..a21691ee4d0 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ModelGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ModelGeneratorTest.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl; import org.junit.Test; import java.util.Collections; @@ -37,12 +38,12 @@ public class ModelGeneratorTest { public void toApplicationModelWithConfigServerApplication() throws Exception { SuperModel superModel = ExampleModel.createExampleSuperModelWithOneRpcPort(HOSTNAME, PORT); - ModelGenerator modelGenerator = new ModelGenerator(); - - Zone zone = new Zone(Environment.from(ENVIRONMENT), RegionName.from(REGION)); List<String> configServerHosts = Stream.of("cfg1", "cfg2", "cfg3") .collect(Collectors.toList()); + ModelGenerator modelGenerator = new ModelGenerator(configServerHosts); + + Zone zone = new Zone(Environment.from(ENVIRONMENT), RegionName.from(REGION)); SlobrokMonitorManagerImpl slobrokMonitorManager = mock(SlobrokMonitorManagerImpl.class); when(slobrokMonitorManager.getStatus(any(), any(), any(), any())) @@ -52,7 +53,6 @@ public class ModelGeneratorTest { modelGenerator.toServiceModel( superModel, zone, - configServerHosts, slobrokMonitorManager); Map<ApplicationInstanceReference, @@ -82,12 +82,10 @@ public class ModelGeneratorTest { public void toApplicationModel() throws Exception { SuperModel superModel = ExampleModel.createExampleSuperModelWithOneRpcPort(HOSTNAME, PORT); - ModelGenerator modelGenerator = new ModelGenerator(); + ModelGenerator modelGenerator = new ModelGenerator(Collections.emptyList()); Zone zone = new Zone(Environment.from(ENVIRONMENT), RegionName.from(REGION)); - List<String> configServerHosts = Collections.emptyList(); - SlobrokMonitorManagerImpl slobrokMonitorManager = mock(SlobrokMonitorManagerImpl.class); when(slobrokMonitorManager.getStatus(any(), any(), any(), any())) .thenReturn(ServiceStatus.UP); @@ -96,7 +94,6 @@ public class ModelGeneratorTest { modelGenerator.toServiceModel( superModel, zone, - configServerHosts, slobrokMonitorManager); Map<ApplicationInstanceReference, diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImplTest.java index 6233f39b9cf..83bad0ddb2a 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImplTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImplTest.java @@ -6,9 +6,9 @@ import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl; import org.junit.Test; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -24,13 +24,11 @@ public class SuperModelListenerImplTest { ServiceMonitorMetrics metrics = mock(ServiceMonitorMetrics.class); ModelGenerator modelGenerator = mock(ModelGenerator.class); Zone zone = mock(Zone.class); - List<String> configServers = new ArrayList<>(); SuperModelListenerImpl listener = new SuperModelListenerImpl( slobrokMonitorManager, metrics, modelGenerator, - zone, - configServers); + zone); SuperModelProvider superModelProvider = mock(SuperModelProvider.class); SuperModel superModel = mock(SuperModel.class); @@ -47,6 +45,6 @@ public class SuperModelListenerImplTest { verify(slobrokMonitorManager).applicationActivated(superModel, application2); ServiceModel serviceModel = listener.get(); - verify(modelGenerator).toServiceModel(superModel, zone, configServers, slobrokMonitorManager); + verify(modelGenerator).toServiceModel(superModel, zone, slobrokMonitorManager); } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManagerTest.java index e557f0451f4..b7c3ed8e1e1 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManagerTest.java @@ -6,6 +6,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.service.monitor.internal.health.HealthMonitorManager; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl; import org.junit.Test; import static com.yahoo.vespa.applicationmodel.ClusterId.NODE_ADMIN; diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorManagerImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorManagerImplTest.java index ab50b3192e3..8e4443df83b 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorManagerImplTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorManagerImplTest.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.internal; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.slobrok; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.SuperModel; diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorTest.java index 075647e9c16..5b230e81cf7 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorTest.java @@ -1,9 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.internal; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.slobrok; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.jrt.slobrok.api.SlobrokList; +import com.yahoo.vespa.service.monitor.internal.ExampleModel; import org.junit.Test; import static org.mockito.Mockito.mock; diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp index 8fd96153dd7..ebeda4f1b8b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp @@ -56,7 +56,7 @@ JSONWriter::quote(const char * str, size_t len) case '\"': case '\\': v[j++] = '\\'; - //@fallthrough@ + [[fallthrough]]; default: v[j++] = str[i]; break; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index e87eabd19df..06dfc073f61 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -569,7 +569,7 @@ FileStorHandlerImpl::remapMessage(api::StorageMessage& msg, const document::Buck } // Follow onto next to move queue or fail } - //@fallthrough@ + [[fallthrough]]; case api::MessageType::SPLITBUCKET_ID: // Move to correct queue if op == MOVE // Fail with bucket not found if op is JOIN @@ -640,7 +640,7 @@ FileStorHandlerImpl::remapMessage(api::StorageMessage& msg, const document::Buck break; case GetIterCommand::ID: bucket = static_cast<GetIterCommand&>(msg).getBucket(); - //@fallthrough@ + [[fallthrough]]; case RepairBucketCommand::ID: if (bucket.getBucketId().getRawId() == 0) { bucket = static_cast<RepairBucketCommand&>(msg).getBucket(); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index 63c957207a6..45ac5ded47f 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -190,8 +190,14 @@ public: std::string dumpQueue() const; void dumpActiveHtml(std::ostream & os) const; void dumpQueueHtml(std::ostream & os) const; + static uint64_t dispersed_bucket_bits(const document::Bucket& bucket) noexcept { + // Disperse bucket bits by multiplying with the 64-bit FNV-1 prime. + // This avoids an inherent affinity between the LSB of a bucket's bits + // and the stripe an operation ends up on. + return bucket.getBucketId().getRawId() * 1099511628211ULL; + } Stripe & stripe(const document::Bucket & bucket) { - return _stripes[bucket.getBucketId().getRawId()%_stripes.size()]; + return _stripes[dispersed_bucket_bits(bucket) % _stripes.size()]; } std::vector<Stripe> & getStripes() { return _stripes; } private: diff --git a/valgrind-suppressions.txt b/valgrind-suppressions.txt index 92954b39f92..baef981a3f9 100644 --- a/valgrind-suppressions.txt +++ b/valgrind-suppressions.txt @@ -35,6 +35,38 @@ fun:main } { + Bug in cppunit. This suppression is created on CentOS7. + Memcheck:Leak + match-leak-kinds: definite + fun:_Znwm + fun:allocate + fun:_S_create + fun:_S_construct<char const*> + fun:_S_construct_aux<char const*> + fun:_S_construct<char const*> + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZN7CppUnit10TestRunnerC1Ev + fun:_ZN7CppUnit14TextTestRunnerC1EPNS_9OutputterE + fun:_ZN10vdstestlib17CppUnitTestRunner3runEiPPKc + fun:main +} +{ + Bug in cppunit. This suppression is created on CentOS7. + Memcheck:Leak + match-leak-kinds: definite + fun:_Znwm + fun:allocate + fun:_S_create + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_S_construct_aux<char const*> + fun:_S_construct<char const*> + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZN7CppUnit10TestRunnerC1Ev + fun:_ZN7CppUnit14TextTestRunnerC1EPNS_9OutputterE + fun:_ZN10vdstestlib17CppUnitTestRunner3runEiPPKc + fun:main +} +{ RHEL6 strlen is eager and will read 16 bytes blocks. Memcheck:Cond fun:__strlen_sse42 diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzService.java index c566d4fe4af..1cf19151b2e 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzService.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.api; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; + import java.util.Objects; /** @@ -20,6 +22,15 @@ public class AthenzService implements AthenzIdentity { this(new AthenzDomain(domain), serviceName); } + public AthenzService(String fullName) { + AthenzIdentity identity = AthenzIdentities.from(fullName); + if (!(identity instanceof AthenzService)) { + throw new IllegalArgumentException(String.format("'%s' is not an Athenz service", fullName)); + } + AthenzService service = (AthenzService) identity; + this.domain = service.getDomain(); + this.serviceName = service.serviceName; + } @Override public AthenzDomain getDomain() { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index fb71ed65da1..ebff56a6f48 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -7,6 +7,8 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.tls.KeyStoreType; import com.yahoo.vespa.athenz.tls.SslContextBuilder; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; +import com.yahoo.vespa.athenz.utils.SiaUtils; import javax.net.ssl.SSLContext; import java.io.File; @@ -42,8 +44,8 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde @Inject public SiaIdentityProvider(SiaProviderConfig config) { this(new AthenzService(config.athenzDomain(), config.athenzService()), - getPrivateKeyFile(config.keyPathPrefix(), config.athenzDomain(), config.athenzService()), - getCertificateFile(config.keyPathPrefix(), config.athenzDomain(), config.athenzService()), + SiaUtils.getPrivateKeyFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())).toFile(), + SiaUtils.getCertificateFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())).toFile(), new File(config.trustStorePath()), createScheduler()); } @@ -52,8 +54,8 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde Path siaPath, File trustStoreFile) { this(service, - getPrivateKeyFile(siaPath.toString(), service.getDomain().getName(), service.getName()), - getCertificateFile(siaPath.toString(), service.getDomain().getName(), service.getName()), + SiaUtils.getPrivateKeyFile(siaPath, service).toFile(), + SiaUtils.getCertificateFile(siaPath, service).toFile(), trustStoreFile, createScheduler()); } @@ -119,13 +121,6 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde } } - private static File getCertificateFile(String rootPath, String domain, String service) { - return Paths.get(rootPath, "certs", String.format("%s.%s.cert.pem", domain, service)).toFile(); - } - - private static File getPrivateKeyFile(String rootPath, String domain, String service) { - return Paths.get(rootPath, "keys", String.format("%s.%s.key.pem", domain, service)).toFile(); - } @Override public void deconstruct() { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java new file mode 100644 index 00000000000..adaafab4617 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -0,0 +1,39 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.utils; + +import com.yahoo.vespa.athenz.api.AthenzService; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Misc utility methods for SIA provided credentials + * + * @author bjorncs + */ +public class SiaUtils { + public static final Path DEFAULT_SIA_DIRECTORY = Paths.get("/var/lib/sia"); + + private SiaUtils() {} + + public static Path getPrivateKeyFile(AthenzService service) { + return getPrivateKeyFile(DEFAULT_SIA_DIRECTORY, service); + } + + public static Path getPrivateKeyFile(Path root, AthenzService service) { + return root + .resolve("keys") + .resolve(String.format("%s.%s.key.pem", service.getDomainName(), service.getName())); + } + + public static Path getCertificateFile(AthenzService service) { + return getCertificateFile(DEFAULT_SIA_DIRECTORY, service); + } + + public static Path getCertificateFile(Path root, AthenzService service) { + return root + .resolve("certs") + .resolve(String.format("%s.%s.cert.pem", service.getDomainName(), service.getName())); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java index 7a59be49458..4390f70cac0 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java @@ -7,7 +7,6 @@ import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.Raw; import com.yahoo.io.ByteWriter; -import com.yahoo.prelude.templates.Context; import com.yahoo.text.XML; import java.io.IOException; @@ -38,7 +37,7 @@ public class DocumentFieldTemplate extends com.yahoo.prelude.templates.UserTempl } @Override - public void error(Context context, Writer writer) throws IOException { + public void error(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { // Error shouldn't be handled by this template, but rather // delegated to the searcher } @@ -55,7 +54,7 @@ public class DocumentFieldTemplate extends com.yahoo.prelude.templates.UserTempl } @Override - public void header(Context context, Writer writer) throws IOException { + public void header(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { if (wrapXml) { // XML wrapping should only be used for default field rendering writer.write("<?xml version=\"1.0\" encoding=\"" + encoding + "\"?>\n"); @@ -64,14 +63,14 @@ public class DocumentFieldTemplate extends com.yahoo.prelude.templates.UserTempl } @Override - public void footer(Context context, Writer writer) throws IOException { + public void footer(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { if (wrapXml) { writer.write("</result>\n"); } } @Override - public void hit(Context context, Writer writer) throws IOException { + public void hit(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { DocumentHit hit = (DocumentHit)context.get("hit"); Document doc = hit.getDocument(); // Assume field existence has been checked before we ever get here. @@ -88,11 +87,11 @@ public class DocumentFieldTemplate extends com.yahoo.prelude.templates.UserTempl } @Override - public void hitFooter(Context context, Writer writer) throws IOException { + public void hitFooter(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { } @Override - public void noHits(Context context, Writer writer) throws IOException { + public void noHits(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java index 25ee0ff5d03..b16f39800ef 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java @@ -6,7 +6,6 @@ import com.yahoo.search.Result; import com.yahoo.search.result.ErrorHit; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.HitGroup; -import com.yahoo.prelude.templates.Context; import com.yahoo.search.result.Hit; import com.yahoo.text.XML; @@ -55,7 +54,7 @@ public class DocumentXMLTemplate extends com.yahoo.prelude.templates.UserTemplat } @Override - public void error(Context context, Writer writer) throws IOException { + public void error(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { writer.write("<errors>\n"); // If the error contains no error hits, use a single error with the main // code and description. Otherwise, use the error hits explicitly @@ -72,7 +71,7 @@ public class DocumentXMLTemplate extends com.yahoo.prelude.templates.UserTemplat } @Override - public void header(Context context, Writer writer) throws IOException { + public void header(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); writer.write("<result>\n"); HitGroup rootGroup = ((Result) context.get("result")).hits(); @@ -82,12 +81,12 @@ public class DocumentXMLTemplate extends com.yahoo.prelude.templates.UserTemplat } @Override - public void footer(Context context, Writer writer) throws IOException { + public void footer(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { writer.write("</result>\n"); } @Override - public void hit(Context context, Writer writer) throws IOException { + public void hit(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { Hit hit = (Hit)context.get("hit"); if (hit instanceof DocumentHit) { DocumentHit docHit = (DocumentHit) hit; @@ -110,11 +109,11 @@ public class DocumentXMLTemplate extends com.yahoo.prelude.templates.UserTemplat } @Override - public void hitFooter(Context context, Writer writer) throws IOException { + public void hitFooter(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { } @Override - public void noHits(Context context, Writer writer) throws IOException { + public void noHits(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index cd491073a1c..5f49dd5ddf8 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -181,9 +181,10 @@ class ClientFeederV3 { if (! operationId.isPresent()) { return Optional.empty(); } - DocumentOperationMessageV3 msg; + + DocumentOperationMessageV3 message; try { - msg = getNextMessage(operationId.get(), requestInputStream, settings); + message = getNextMessage(operationId.get(), requestInputStream, settings); } catch (Exception e) { if (log.isLoggable(LogLevel.DEBUG)) { log.log(LogLevel.DEBUG, Exceptions.toMessageString(e), e); @@ -193,8 +194,9 @@ class ClientFeederV3 { continue; } - setRoute(msg, settings); - return Optional.of(msg); + if (message != null) + setRoute(message, settings); + return Optional.ofNullable(message); } } @@ -271,6 +273,7 @@ class ClientFeederV3 { } // protected for mocking + /** Returns the next message in the stream, or null if none */ protected DocumentOperationMessageV3 getNextMessage( String operationId, InputStream requestInputStream, FeederSettings settings) throws Exception { VespaXMLFeedReader.Operation operation = streamReaderV3.getNextOperation(requestInputStream, settings); @@ -283,14 +286,14 @@ class ClientFeederV3 { null); } - DocumentOperationMessageV3 msg = DocumentOperationMessageV3.create(operation, operationId, metric); - if (msg == null) { + DocumentOperationMessageV3 message = DocumentOperationMessageV3.create(operation, operationId, metric); + if (message == null) { // typical end of feed return null; } metric.add(MetricNames.NUM_OPERATIONS, 1, null /*metricContext*/); - log(LogLevel.DEBUG, "Successfully deserialized document id: ", msg.getOperationId()); - return msg; + log(LogLevel.DEBUG, "Successfully deserialized document id: ", message.getOperationId()); + return message; } private void setMessageParameters(DocumentOperationMessageV3 msg, FeederSettings settings) { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java index 2424ce596a3..9d6c8c2feac 100755 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java @@ -17,11 +17,9 @@ import com.yahoo.feedapi.FeedContext; import com.yahoo.feedapi.MessagePropertyProcessor; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.routing.Route; -import com.yahoo.prelude.templates.SearchRendererAdaptor; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.searchchain.Execution; @@ -740,7 +738,7 @@ public class GetSearcherTestCase { assertEquals("application/octet-stream", result.getTemplating().getTemplates().getMimeType()); ByteArrayOutputStream stream = new ByteArrayOutputStream(); - SearchRendererAdaptor.callRender(stream, result); + com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); stream.flush(); byte[] resultBytes = stream.toByteArray(); @@ -769,7 +767,7 @@ public class GetSearcherTestCase { assertEquals("text/fancy", result.getTemplating().getTemplates().getMimeType()); ByteArrayOutputStream stream = new ByteArrayOutputStream(); - SearchRendererAdaptor.callRender(stream, result); + com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); stream.flush(); byte[] resultBytes = stream.toByteArray(); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java index 1c855455a37..d42faf418a1 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java @@ -33,7 +33,6 @@ public class BucketStatsRetriever { private final MessageBusSyncSession session; private final MessageBusDocumentAccess documentAccess; - private final String route; public BucketStatsRetriever( DocumentAccessFactory documentAccessFactory, @@ -42,7 +41,7 @@ public class BucketStatsRetriever { registerShutdownHook(registrar); this.documentAccess = documentAccessFactory.createDocumentAccess(); this.session = documentAccess.createSyncSession(new SyncParameters.Builder().build()); - this.route = route; + this.session.setRoute(route); } private void registerShutdownHook(ShutdownHookRegistrar registrar) { @@ -102,19 +101,10 @@ public class BucketStatsRetriever { private <T extends Reply> T sendMessage(DocumentMessage msg, Class<T> expectedReply) throws BucketStatsException { - setRoute(msg, route); Reply reply = session.syncSend(msg); return validateReply(reply, expectedReply); } - private static void setRoute(DocumentMessage msg, String route) throws BucketStatsException { - try { - msg.setRoute(Route.parse(route)); - } catch (Exception e) { - throw new BucketStatsException(String.format("Invalid route: '%s'.", route)); - } - } - private static <T extends Reply> T validateReply(Reply reply, Class<T> type) throws BucketStatsException { if (reply.hasErrors()) { throw new BucketStatsException(makeErrorMessage(reply)); diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java index a34e3a73205..475547546d3 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java @@ -20,6 +20,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -127,12 +128,8 @@ public class BucketStatsRetrieverTest { BucketStatsRetriever retriever = new BucketStatsRetriever(mockedFactory, route, t -> {}); retriever.retrieveBucketList(new BucketId(0), bucketSpace); - verify(mockedSession).syncSend(argThat(new ArgumentMatcher<Message>() { - @Override - public boolean matches(Object o) { - return ((Message) o).getRoute().equals(Route.parse(route)); - } - })); + // Route is set at session-level, not per message sent. + verify(mockedSession).setRoute(eq(route)); } private BucketStatsRetriever createRetriever() { diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp index 94e214e9fb9..366111cad0d 100644 --- a/vespalib/src/tests/stllike/hash_test.cpp +++ b/vespalib/src/tests/stllike/hash_test.cpp @@ -434,6 +434,45 @@ TEST("test that for_each member works as std::for_each") { TEST_DO(verify_sum(m, expected_sum)); } +namespace { + +class WrappedKey +{ + std::unique_ptr<const int> _key; +public: + WrappedKey() : _key() { } + WrappedKey(int key) : _key(std::make_unique<const int>(key)) { } + size_t hash() const { return vespalib::hash<int>()(*_key); } + bool operator==(const WrappedKey &rhs) const { return *_key == *rhs._key; } +}; + +} + +TEST("test that hash map can have non-copyable key") +{ + hash_map<WrappedKey, int> m; + EXPECT_TRUE(m.insert(std::make_pair(WrappedKey(4), 5)).second); + WrappedKey testKey(4); + ASSERT_TRUE(m.find(testKey) != m.end()); + EXPECT_EQUAL(5, m.find(testKey)->second); +} + +TEST("test that hash map can have non-copyable value") +{ + hash_map<int, std::unique_ptr<int>> m; + EXPECT_TRUE(m.insert(std::make_pair(4, std::make_unique<int>(5))).second); + EXPECT_TRUE(m[4]); + EXPECT_EQUAL(5, *m[4]); +} + +TEST("test that hash set can have non-copyable key") +{ + hash_set<WrappedKey> m; + EXPECT_TRUE(m.insert(WrappedKey(4)).second); + WrappedKey testKey(4); + ASSERT_TRUE(m.find(testKey) != m.end()); +} + using IntHashSet = hash_set<int>; TEST("test hash set initializer list - empty") diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp index 18bf5289f0d..72b494e2479 100644 --- a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp @@ -422,7 +422,7 @@ JsonDecoder::decodeNumber(Inserter &inserter) switch (c) { case '+': case '-': case '.': case 'e': case 'E': isLong = false; - //@fallthrough@ + [[fallthrough]]; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': value.push_back(c); diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.h b/vespalib/src/vespa/vespalib/stllike/hash_map.h index f2431a73f28..6d6498f8e78 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.h @@ -36,6 +36,7 @@ public: size_t size() const { return _ht.size(); } bool empty() const { return _ht.empty(); } insert_result insert(const value_type & value) { return _ht.insert(value); } + insert_result insert(value_type &&value) { return _ht.insert(std::move(value)); } template <typename InputIt> void insert(InputIt first, InputIt last); diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.h b/vespalib/src/vespa/vespalib/stllike/hash_set.h index bb16932a990..c4ccc662787 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_set.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.h @@ -37,6 +37,7 @@ public: size_t size() const { return _ht.size(); } bool empty() const { return _ht.empty(); } insert_result insert(const K & value); + insert_result insert(K &&value); template<typename InputIt> void insert(InputIt first, InputIt last); void erase(const K & key); diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.hpp b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp index bd323b2c860..cf6341218f1 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_set.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp @@ -74,6 +74,12 @@ hash_set<K, H, EQ, M>::insert(const K & value) { return _ht.insert(value); } +template<typename K, typename H, typename EQ, typename M> +typename hash_set<K, H, EQ, M>::insert_result +hash_set<K, H, EQ, M>::insert(K &&value) { + return _ht.insert(std::move(value)); +} + } #define VESPALIB_HASH_SET_INSTANTIATE(K) \ diff --git a/vespalib/src/vespa/vespalib/util/bobhash.h b/vespalib/src/vespa/vespalib/util/bobhash.h index 2aff09929b2..60cbe2cbca3 100644 --- a/vespalib/src/vespa/vespalib/util/bobhash.h +++ b/vespalib/src/vespa/vespalib/util/bobhash.h @@ -128,18 +128,18 @@ public: c += length; switch(len) /* all the case statements fall through */ { - case 11: c += (static_cast<uint32_t>(k[10]) << 24); //@fallthrough@ - case 10: c += (static_cast<uint32_t>(k[9]) << 16); //@fallthrough@ - case 9 : c += (static_cast<uint32_t>(k[8]) << 8); //@fallthrough@ + case 11: c += (static_cast<uint32_t>(k[10]) << 24); [[fallthrough]]; + case 10: c += (static_cast<uint32_t>(k[9]) << 16); [[fallthrough]]; + case 9 : c += (static_cast<uint32_t>(k[8]) << 8); [[fallthrough]]; /* the first byte of c is reserved for the length */ - case 8 : b += (static_cast<uint32_t>(k[7]) << 24); //@fallthrough@ - case 7 : b += (static_cast<uint32_t>(k[6]) << 16); //@fallthrough@ - case 6 : b += (static_cast<uint32_t>(k[5]) << 8); //@fallthrough@ - case 5 : b += k[4]; //@fallthrough@ - case 4 : a += (static_cast<uint32_t>(k[3]) << 24); //@fallthrough@ - case 3 : a += (static_cast<uint32_t>(k[2]) << 16); //@fallthrough@ - case 2 : a += (static_cast<uint32_t>(k[1]) << 8); //@fallthrough@ - case 1 : a += k[0]; //@fallthrough@ + case 8 : b += (static_cast<uint32_t>(k[7]) << 24); [[fallthrough]]; + case 7 : b += (static_cast<uint32_t>(k[6]) << 16); [[fallthrough]]; + case 6 : b += (static_cast<uint32_t>(k[5]) << 8); [[fallthrough]]; + case 5 : b += k[4]; [[fallthrough]]; + case 4 : a += (static_cast<uint32_t>(k[3]) << 24); [[fallthrough]]; + case 3 : a += (static_cast<uint32_t>(k[2]) << 16); [[fallthrough]]; + case 2 : a += (static_cast<uint32_t>(k[1]) << 8); [[fallthrough]]; + case 1 : a += k[0]; /* case 0: nothing left to add */ } bobhash_mix(a,b,c); diff --git a/vespalog/src/logctl/logctl.cpp b/vespalog/src/logctl/logctl.cpp index 111dfa071bb..a0963b43c34 100644 --- a/vespalog/src/logctl/logctl.cpp +++ b/vespalog/src/logctl/logctl.cpp @@ -116,10 +116,10 @@ main(int argc, char **argv) break; case 'r': doResetLevels = true; - //@fallthrough@ + [[fallthrough]]; case 'c': shouldCreateFile = true; - //@fallthrough@ + [[fallthrough]]; case 'n': shouldCreateEntry = true; break; diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp index bd57c05f8ac..21fa1d9ed02 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp @@ -197,7 +197,7 @@ DocsumFilter::getFieldValue(const DocsumFieldSpec::FieldIdentifier & fieldId, return _cachedValue.get(); } } - //@fallthrough@ + [[fallthrough]]; default: return fv; } diff --git a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp index 8855923610c..b21177a6810 100644 --- a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp +++ b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp @@ -65,7 +65,7 @@ FieldSearchSpec::FieldSearchSpec(const FieldIdT & fid, const vespalib::string & switch(searchDef) { default: LOG(warning, "Unknown searchdef = %d. Defaulting to AUTOUTF8", searchDef); - //@fallthrough@ + [[fallthrough]]; case VsmfieldsConfig::Fieldspec::AUTOUTF8: case VsmfieldsConfig::Fieldspec::NONE: case VsmfieldsConfig::Fieldspec::SSE2UTF8: diff --git a/yolean/src/main/java/com/yahoo/yolean/Exceptions.java b/yolean/src/main/java/com/yahoo/yolean/Exceptions.java index 83c381e586f..82677a14242 100644 --- a/yolean/src/main/java/com/yahoo/yolean/Exceptions.java +++ b/yolean/src/main/java/com/yahoo/yolean/Exceptions.java @@ -1,6 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.yolean; +import java.io.IOException; +import java.io.UncheckedIOException; + /** * Helper methods for handling exceptions * @@ -44,4 +47,59 @@ public class Exceptions { return message; } + /** + * Wraps any IOException thrown from a runnable in an UncheckedIOException. + */ + public static void uncheck(RunnableThrowingIOException runnable) { + try { + runnable.run(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Wraps any IOException thrown from a runnable in an UncheckedIOException w/message. + */ + public static void uncheck(RunnableThrowingIOException runnable, String format, String... args) { + try { + runnable.run(); + } catch (IOException e) { + String message = String.format(format, (Object[]) args); + throw new UncheckedIOException(message, e); + } + } + + @FunctionalInterface + public interface RunnableThrowingIOException { + void run() throws IOException; + } + + /** + * Wraps any IOException thrown from a supplier in an UncheckedIOException. + */ + public static <T> T uncheck(SupplierThrowingIOException<T> supplier) { + try { + return supplier.get(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Wraps any IOException thrown from a supplier in an UncheckedIOException w/message. + */ + public static <T> T uncheck(SupplierThrowingIOException<T> supplier, String format, String... args) { + try { + return supplier.get(); + } catch (IOException e) { + String message = String.format(format, (Object[]) args); + throw new UncheckedIOException(message, e); + } + } + + @FunctionalInterface + public interface SupplierThrowingIOException<T> { + T get() throws IOException; + } } diff --git a/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java b/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java index db605609926..31e27fa2675 100644 --- a/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java +++ b/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java @@ -3,6 +3,9 @@ package com.yahoo.yolean; import org.junit.Test; +import java.io.IOException; +import java.io.UncheckedIOException; + import static org.junit.Assert.assertEquals; /** @@ -21,4 +24,38 @@ public class ExceptionsTestCase { assertEquals("Foo",Exceptions.toMessageString(new Exception(new Exception("Foo")))); } + @Test + public void testUnchecks() { + try { + Exceptions.uncheck(this::throwIO); + } catch (UncheckedIOException e) { + assertEquals("root cause", e.getCause().getMessage()); + } + + try { + Exceptions.uncheck(this::throwIO, "additional %s", "info"); + } catch (UncheckedIOException e) { + assertEquals("additional info", e.getMessage()); + } + + try { + int i = Exceptions.uncheck(this::throwIOWithReturnValue); + } catch (UncheckedIOException e) { + assertEquals("root cause", e.getCause().getMessage()); + } + + try { + int i = Exceptions.uncheck(this::throwIOWithReturnValue, "additional %s", "info"); + } catch (UncheckedIOException e) { + assertEquals("additional info", e.getMessage()); + } + } + + private void throwIO() throws IOException { + throw new IOException("root cause"); + } + + private int throwIOWithReturnValue() throws IOException { + throw new IOException("root cause"); + } } |