diff options
120 files changed, 730 insertions, 1640 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 0213e753217..e873432f71d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -54,6 +54,7 @@ public interface ModelContext { boolean isFirstTimeDeployment(); boolean useDedicatedNodeForLogserver(); boolean useFdispatchByDefault(); + boolean useAdaptiveDispatch(); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index a6f2a205f94..f93688abddc 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -36,6 +36,7 @@ public class TestProperties implements ModelContext.Properties { private boolean isFirstTimeDeployment = false; private boolean useDedicatedNodeForLogserver = false; private boolean useFdispatchByDefault = true; + private boolean useAdaptiveDispatch = false; @Override public boolean multitenant() { return multitenant; } @Override public ApplicationId applicationId() { return applicationId; } @@ -48,6 +49,7 @@ public class TestProperties implements ModelContext.Properties { @Override public Set<Rotation> rotations() { return rotations; } @Override public boolean isBootstrap() { return isBootstrap; } @Override public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } + @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @Override public boolean useFdispatchByDefault() { return useFdispatchByDefault; } @@ -61,6 +63,11 @@ public class TestProperties implements ModelContext.Properties { return this; } + public TestProperties setUseAdaptiveDispatch(boolean useAdaptiveDispatch) { + this.useAdaptiveDispatch = useAdaptiveDispatch; + return this; + } + public TestProperties setMultitenant(boolean multitenant) { this.multitenant = multitenant; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java index b3815994742..ec958589483 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java @@ -33,7 +33,7 @@ public class TuningDispatch { public static class Builder { private Integer maxHitsPerPartition; - private DispatchPolicy dispatchPolicy = DispatchPolicy.ROUNDROBIN; + private DispatchPolicy dispatchPolicy; private Boolean useLocalNode; private Double minGroupCoverage; private Double minActiveDocsCoverage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java index 8158de692a9..b9c937b4a4c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.content.SearchCoverage; -import com.yahoo.vespa.model.content.TuningDispatch; import java.util.ArrayList; import java.util.List; @@ -32,6 +31,7 @@ public class Dispatch extends AbstractService implements SearchInterface, private final int dispatchLevel; private final boolean preferLocalRow; private final boolean isTopLevel; + private boolean useAdaptiveDispatch; private Dispatch(DispatchGroup dispatchGroup, AbstractConfigProducer parent, String subConfigId, NodeSpec nodeSpec, int dispatchLevel, boolean preferLocalRow, boolean isTopLevel) { @@ -41,6 +41,7 @@ public class Dispatch extends AbstractService implements SearchInterface, this.dispatchLevel = dispatchLevel; this.preferLocalRow = preferLocalRow; this.isTopLevel = isTopLevel; + this.useAdaptiveDispatch = false; portsMeta.on(0).tag("rpc").tag("admin"); portsMeta.on(1).tag("fs4"); portsMeta.on(2).tag("http").tag("json").tag("health").tag("state"); @@ -49,6 +50,11 @@ public class Dispatch extends AbstractService implements SearchInterface, .setProp("index", nodeSpec.groupIndex()); } + public Dispatch useAdaptiveDispatch(boolean useAdaptiveDispatch) { + this.useAdaptiveDispatch = useAdaptiveDispatch; + return this; + } + public static Dispatch createTld(DispatchGroup dispatchGroup, AbstractConfigProducer parent, int rowId) { return createTld(dispatchGroup, parent, rowId, false); } @@ -139,6 +145,9 @@ public class Dispatch extends AbstractService implements SearchInterface, datasetBuilder.querydistribution(PartitionsConfig.Dataset.Querydistribution.Enum.FIXEDROW); datasetBuilder.maxnodesdownperfixedrow(dispatchGroup.getMaxNodesDownPerFixedRow()); } + if (useAdaptiveDispatch) { + datasetBuilder.useroundrobinforfixedrow(false); + } SearchCoverage coverage = dispatchGroup.getSearchCoverage(); if (coverage != null) { if (coverage.getMinimum() != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 6aa023e486b..fe61041196b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -107,6 +107,7 @@ public class IndexedSearchCluster extends SearchCluster private final DispatchGroup rootDispatch; private DispatchSpec dispatchSpec; private final boolean useFdispatchByDefault; + private final boolean useAdaptiveDispatch; private List<SearchNode> searchNodes = new ArrayList<>(); /** @@ -125,6 +126,7 @@ public class IndexedSearchCluster extends SearchCluster dispatchParent = new SimpleConfigProducer(this, "dispatchers"); rootDispatch = new DispatchGroup(this); useFdispatchByDefault = deployState.getProperties().useFdispatchByDefault(); + useAdaptiveDispatch = deployState.getProperties().useAdaptiveDispatch(); } @Override @@ -186,6 +188,7 @@ public class IndexedSearchCluster extends SearchCluster public Dispatch addTld(DeployLogger deployLogger, AbstractConfigProducer tldParent, HostResource hostResource) { int index = rootDispatch.getDispatchers().size(); Dispatch tld = Dispatch.createTld(rootDispatch, tldParent, index); + tld.useAdaptiveDispatch(useAdaptiveDispatch); tld.setHostResource(hostResource); tld.initService(deployLogger); rootDispatch.addDispatcher(tld); @@ -210,12 +213,13 @@ public class IndexedSearchCluster extends SearchCluster log.log(LogLevel.DEBUG, "Adding tld with index " + containerIndex + " for content cluster " + this.getClusterName() + ", container cluster " + containerClusterName + " (container id " + containerSubId + ") on host " + container.getHostResource().getHostname()); - rootDispatch.addDispatcher(createTld(deployLogger, tldParent, container.getHostResource(), containerClusterName, containerIndex)); + rootDispatch.addDispatcher(createTld(deployLogger, tldParent, container.getHostResource(), containerClusterName, containerIndex).useAdaptiveDispatch(useAdaptiveDispatch)); } } private Dispatch createTld(DeployLogger deployLogger, AbstractConfigProducer tldParent, HostResource hostResource, String containerClusterName, int containerIndex) { Dispatch tld = Dispatch.createTldWithContainerIdInName(rootDispatch, tldParent, containerClusterName, containerIndex); + tld.useAdaptiveDispatch(useAdaptiveDispatch); tld.setHostResource(hostResource); tld.initService(deployLogger); return tld; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index 2571381167e..2e51e432e68 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -19,7 +19,7 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P public static class Dispatch implements PartitionsConfig.Producer { public Integer maxHitsPerPartition = null; - public TuningDispatch.DispatchPolicy policy = TuningDispatch.DispatchPolicy.ROUNDROBIN; + public TuningDispatch.DispatchPolicy policy = null; public boolean useLocalNode = false; public Double minGroupCoverage = null; public Double minActiveDocsCoverage = null; @@ -41,15 +41,17 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P dataset.min_activedocs_coverage(minActiveDocsCoverage); } } - for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { - switch (policy) { - case ADAPTIVE: - dataset.useroundrobinforfixedrow(false); - break; - case ROUNDROBIN: - default: - dataset.useroundrobinforfixedrow(true); - break; + if (policy != null) { + for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { + switch (policy) { + case ADAPTIVE: + dataset.useroundrobinforfixedrow(false); + break; + case ROUNDROBIN: + default: + dataset.useroundrobinforfixedrow(true); + break; + } } } } diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java index 18d017d76f0..38e438e4d3a 100644 --- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java +++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java @@ -6,25 +6,16 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.Rotation; -import com.yahoo.config.provision.Zone; - -import java.net.URI; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; + import java.util.Optional; -import java.util.Set; /** * @author hmusum @@ -84,61 +75,6 @@ public class MockModelContext implements ModelContext { @Override public Properties properties() { - return new Properties() { - @Override - public boolean multitenant() { - return false; - } - - @Override - public ApplicationId applicationId() { - return ApplicationId.defaultId(); - } - - @Override - public List<ConfigServerSpec> configServerSpecs() { - return Collections.emptyList(); - } - - @Override - public HostName loadBalancerName() { - return null; - } - - @Override - public URI ztsUrl() { - return null; - } - - @Override - public String athenzDnsSuffix() { - return null; - } - - @Override - public boolean hostedVespa() {return false; } - - @Override - public Zone zone() { - return Zone.defaultZone(); - } - - @Override - public Set<Rotation> rotations() { - return new HashSet<>(); - } - - @Override - public boolean isBootstrap() { return false; } - - @Override - public boolean isFirstTimeDeployment() { return false; } - - @Override - public boolean useDedicatedNodeForLogserver() { return false; } - - @Override - public boolean useFdispatchByDefault() { return true; } - }; + return new TestProperties(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index e2337c42cf3..0e200efd688 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockRoot; @@ -16,6 +17,7 @@ import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.config.search.core.PartitionsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ContainerCluster; @@ -177,7 +179,7 @@ public class ContentClusterTest extends ContentBaseTest { } @Test - public void testEndToEnd() throws Exception { + public void testEndToEnd() { String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<services>\n" + @@ -220,49 +222,73 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(3, cluster.getContainers().size()); } - @Test - public void testEndToEndOneNode() throws Exception { + VespaModel createEnd2EndOneNode(ModelContext.Properties properties) { String services = - "<?xml version='1.0' encoding='UTF-8' ?>" + - "<services version='1.0'>" + - " <admin version='2.0'>" + - " <adminserver hostalias='node1'/>" + - " </admin>" + - " <jdisc id='default' version='1.0'>" + - " <search/>" + - " <nodes>" + - " <node hostalias='node1'/>" + - " </nodes>" + - " </jdisc>" + - " <content id='storage' version='1.0'>" + - " <redundancy>2</redundancy>" + - " <group>" + - " <node distribution-key='0' hostalias='node1'/>" + - " <node distribution-key='1' hostalias='node1'/>" + - " </group>" + - " <tuning>" + - " <cluster-controller>" + - " <transition-time>0</transition-time>" + - " </cluster-controller>" + - " </tuning>" + - " <documents>" + - " <document mode='store-only' type='type1'/>" + - " </documents>" + - " <engine>" + - " <proton/>" + - " </engine>" + - " </content>" + - " </services>"; - + "<?xml version='1.0' encoding='UTF-8' ?>" + + "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node1'/>" + + " </admin>" + + " <jdisc id='default' version='1.0'>" + + " <search/>" + + " <nodes>" + + " <node hostalias='node1'/>" + + " </nodes>" + + " </jdisc>" + + " <content id='storage' version='1.0'>" + + " <redundancy>2</redundancy>" + + " <group>" + + " <node distribution-key='0' hostalias='node1'/>" + + " <node distribution-key='1' hostalias='node1'/>" + + " </group>" + + " <tuning>" + + " <cluster-controller>" + + " <transition-time>0</transition-time>" + + " </cluster-controller>" + + " </tuning>" + + " <documents>" + + " <document mode='index' type='type1'/>" + + " </documents>" + + " <engine>" + + " <proton/>" + + " </engine>" + + " </content>" + + " </services>"; + + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(properties); List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1"); - VespaModel model = (new VespaModelCreatorWithMockPkg(null, services, sds)).create(); + return (new VespaModelCreatorWithMockPkg(null, services, sds)).create(deployStateBuilder); + } + @Test + public void testEndToEndOneNode() { + VespaModel model = createEnd2EndOneNode(new TestProperties()); + assertEquals(1, model.getContentClusters().get("storage").getDocumentDefinitions().size()); ContainerCluster cluster = model.getAdmin().getClusterControllers(); assertEquals(1, cluster.getContainers().size()); } + private void verifyRoundRobinPropertiesControl(boolean useAdaptiveDispatch) { + VespaModel model = createEnd2EndOneNode(new TestProperties().setUseAdaptiveDispatch(useAdaptiveDispatch)); + + ContentCluster cc = model.getContentClusters().get("storage"); + PartitionsConfig.Builder partBuilder = new PartitionsConfig.Builder(); + assertNotNull(cc.getSearch()); + assertNotNull(cc.getSearch().getIndexed()); + assertEquals(1, cc.getSearch().getIndexed().getTLDs().size()); + cc.getSearch().getIndexed().getTLDs().get(0).getConfig(partBuilder); + PartitionsConfig partitionsConfig = new PartitionsConfig(partBuilder); + assertFalse(useAdaptiveDispatch == partitionsConfig.dataset(0).useroundrobinforfixedrow()); + } + + @Test + public void default_dispatch_controlled_by_properties() { + verifyRoundRobinPropertiesControl(false); + verifyRoundRobinPropertiesControl(true); + } + @Test - public void testSearchTuning() throws Exception { + public void testSearchTuning() { String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<services>\n" + @@ -312,7 +338,7 @@ public class ContentClusterTest extends ContentBaseTest { } @Test - public void testRedundancyRequired() throws Exception { + public void testRedundancyRequired() { String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<services>\n" + @@ -754,8 +780,6 @@ public class ContentClusterTest extends ContentBaseTest { " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + " </group>" + "</content>", isHostedVespa); - - } private static ContentCluster createClusterWithFlushOnShutdownOverride(boolean flushOnShutdown, boolean isHostedVespa) throws Exception { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/TuningDispatchTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/TuningDispatchTest.java index e84f256d6dc..fe00648317a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/TuningDispatchTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/TuningDispatchTest.java @@ -49,6 +49,6 @@ public class TuningDispatchTest { public void requireThatDefaultsAreNull() { TuningDispatch dispatch = new TuningDispatch.Builder().build(); assertNull(dispatch.getMaxHitsPerPartition()); - assertTrue(TuningDispatch.DispatchPolicy.ROUNDROBIN == dispatch.getDispatchPolicy()); + assertNull(dispatch.getDispatchPolicy()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 1a2eb93face..44986bc0b69 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -28,7 +28,7 @@ import static org.junit.Assert.assertTrue; public class ClusterTest { @Test - public void requireThatContentSearchIsApplied() throws ParseException { + public void requireThatContentSearchIsApplied() { ContentCluster cluster = newContentCluster(joinLines("<search>", " <query-timeout>1.1</query-timeout>", " <visibility-delay>2.3</visibility-delay>", @@ -64,7 +64,7 @@ public class ClusterTest { } @Test - public void requireThatDispatchTuningIsApplied() throws ParseException { + public void requireThatDispatchTuningIsApplied() { ContentCluster cluster = newContentCluster(joinLines("<search>", "</search>"), joinLines("<tuning>", "</tuning>")); @@ -79,7 +79,7 @@ public class ClusterTest { } @Test - public void requireThatVisibilityDelayIsZeroForGlobalDocumentType() throws ParseException { + public void requireThatVisibilityDelayIsZeroForGlobalDocumentType() { ContentCluster cluster = newContentCluster(joinLines("<search>", " <visibility-delay>2.3</visibility-delay>", "</search>"), true); @@ -87,19 +87,19 @@ public class ClusterTest { assertEquals(0.0, proton.documentdb(0).visibilitydelay(), 1E-6); } - private static ContentCluster newContentCluster(String contentSearchXml) throws ParseException { + private static ContentCluster newContentCluster(String contentSearchXml) { return newContentCluster(contentSearchXml, "", false); } - private static ContentCluster newContentCluster(String contentSearchXml, String searchNodeTuningXml) throws ParseException { + private static ContentCluster newContentCluster(String contentSearchXml, String searchNodeTuningXml) { return newContentCluster(contentSearchXml, searchNodeTuningXml, false); } - private static ContentCluster newContentCluster(String contentSearchXml, boolean globalDocType) throws ParseException { + private static ContentCluster newContentCluster(String contentSearchXml, boolean globalDocType) { return newContentCluster(contentSearchXml, "", globalDocType); } - private static ContentCluster newContentCluster(String contentSearchXml, String searchNodeTuningXml, boolean globalDocType) throws ParseException { + private static ContentCluster newContentCluster(String contentSearchXml, String searchNodeTuningXml, boolean globalDocType) { ApplicationPackage app = new MockApplicationPackage.Builder() .withHosts(joinLines( "<hosts>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java index d9646c163e4..46b5b630f9e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java @@ -45,7 +45,7 @@ public class DomTuningDispatchBuilderTest { assertNull(dispatch.getMaxHitsPerPartition()); assertNull(dispatch.getMinGroupCoverage()); assertNull(dispatch.getMinActiveDocsCoverage()); - assertTrue(TuningDispatch.DispatchPolicy.ROUNDROBIN == dispatch.getDispatchPolicy()); + assertNull(dispatch.getDispatchPolicy()); } @Test diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 899fd83eca1..13612a91726 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -34,7 +34,7 @@ <intVal>1</intVal> </config> - <jdisc id='qrsCluster_1' version='1.0'> + <container id='qrsCluster_1' version='1.0'> <secret-store type="oath-ckms"> <!-- NOTE: when removing (or adding) an environment, the rnc schema must also be updated! --> <group name="foo" environment="alpha" /> @@ -206,14 +206,14 @@ </server> <nodes jvm-options="-XX:+PrintGCDetails -XX:+PrintGCTimeStamps"> + <environment-variables> + <TEST_VAR>7</TEST_VAR> + </environment-variables> <node hostalias="host1" /> <node hostalias="host1"> <server-port id="myServer" port="4090" /> </node> - <environment-variables> - <TEST_VAR>7</TEST_VAR> - </environment-variables> </nodes> - </jdisc> + </container> </services> diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 6274ec77e01..16d1cb6855b 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -4,17 +4,18 @@ package com.yahoo.vespa.config.proxy; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Spec; - import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.log.LogLevel; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; -import com.yahoo.system.CatchSigTerm; -import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.filedistribution.FileDistributionRpcServer; import com.yahoo.vespa.filedistribution.FileDownloader; +import com.yahoo.yolean.system.CatchSignals; import java.util.List; import java.util.concurrent.Executors; @@ -175,8 +176,8 @@ public class ProxyServer implements Runnable { return new RpcConfigSourceClient(configSource, clientUpdater, memoryCache, timingValues, delayedResponses); } - private void setupSigTermHandler() { - CatchSigTerm.setup(signalCaught); // catch termination signal + private void setupSignalHandler() { + CatchSignals.setup(signalCaught); // catch termination and interrupt signals } private void waitForShutdown() { @@ -215,8 +216,8 @@ public class ProxyServer implements Runnable { DelayedResponses delayedResponses = new DelayedResponses(statistics); ProxyServer proxyServer = new ProxyServer(new Spec(null, port), delayedResponses, configSources, statistics, defaultTimingValues(), true, new MemoryCache(), null); - // catch termination signal - proxyServer.setupSigTermHandler(); + // catch termination and interrupt signal + proxyServer.setupSignalHandler(); Thread proxyserverThread = new Thread(proxyServer); proxyserverThread.setName("configproxy"); proxyserverThread.start(); diff --git a/config/src/vespa/config/common/misc.cpp b/config/src/vespa/config/common/misc.cpp index c0f8c96f8d4..1040962e25c 100644 --- a/config/src/vespa/config/common/misc.cpp +++ b/config/src/vespa/config/common/misc.cpp @@ -37,7 +37,7 @@ calculateContentMd5(const std::vector<vespalib::string> & fileContents) if (md5sum[i] < 16) { s << "0"; } - s << std::hex << (int)md5sum[i]; + s << vespalib::hex << (int)md5sum[i]; } return s.str(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 658ffc55772..870a91f8b45 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -130,6 +130,7 @@ public class ModelContextImpl implements ModelContext { private final boolean isFirstTimeDeployment; private final boolean useDedicatedNodeForLogserver; private final boolean useFdispatchByDefault; + private final boolean useAdaptiveDispatch; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -158,6 +159,8 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useFdispatchByDefault = Flags.USE_FDISPATCH_BY_DEFAULT.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); } @Override @@ -202,6 +205,8 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useFdispatchByDefault() { return useFdispatchByDefault; } + @Override + public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } } } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index e402adbaeb9..24a46602b60 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -352,6 +352,34 @@ ], "fields": [] }, + "com.yahoo.prelude.query.BoolItem": { + "superClass": "com.yahoo.prelude.query.TermItem", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(boolean)", + "public void <init>(boolean, java.lang.String)", + "public void <init>(boolean, java.lang.String, boolean)", + "public com.yahoo.prelude.query.Item$ItemType getItemType()", + "public java.lang.String getName()", + "protected void encodeThis(java.nio.ByteBuffer)", + "public boolean value()", + "public java.lang.String stringValue()", + "public void setValue(boolean)", + "public void setValue(java.lang.String)", + "public java.lang.String getRawWord()", + "public boolean isStemmed()", + "protected void appendHeadingString(java.lang.StringBuilder)", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public int getNumWords()", + "public java.lang.String getIndexedString()", + "public boolean isWords()" + ], + "fields": [] + }, "com.yahoo.prelude.query.CompositeIndexedItem": { "superClass": "com.yahoo.prelude.query.CompositeTaggableItem", "interfaces": [ @@ -1321,6 +1349,7 @@ "public" ], "methods": [ + "public void <init>(java.lang.String)", "public void <init>(int, int, java.lang.String)", "public java.lang.String getValue()", "public java.lang.String getSuperstring()", @@ -1618,8 +1647,6 @@ "public" ], "methods": [ - "public com.yahoo.prelude.query.Item$ItemType getItemType()", - "public java.lang.String getName()", "public void <init>(java.lang.String)", "public void <init>(java.lang.String, java.lang.String)", "public void <init>(java.lang.String, boolean)", @@ -1627,6 +1654,8 @@ "public void <init>(com.yahoo.prelude.query.parser.Token, boolean)", "public void <init>(java.lang.String, boolean, com.yahoo.prelude.query.Substring)", "public void <init>(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)", + "public com.yahoo.prelude.query.Item$ItemType getItemType()", + "public java.lang.String getName()", "public void setWord(java.lang.String)", "protected void encodeThis(java.nio.ByteBuffer)", "protected java.lang.String getEncodedWord()", @@ -1728,6 +1757,7 @@ "public void attachContext(com.yahoo.search.Query)", "public java.lang.String yqlRepresentation()", "public java.lang.String yqlRepresentation(com.yahoo.collections.Tuple2, boolean)", + "public java.lang.String yqlRepresentation(boolean)", "public com.yahoo.search.query.context.QueryContext getContext(boolean)", "public int hashCode()", "public boolean equals(java.lang.Object)", diff --git a/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java b/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java new file mode 100644 index 00000000000..27045629780 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java @@ -0,0 +1,102 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import java.nio.ByteBuffer; + +/** + * A true/false term suitable for searching bool indexes. + */ +public class BoolItem extends TermItem { + + private boolean value; + + public BoolItem(boolean value) { + this(value, ""); + } + + public BoolItem(boolean value, String indexName) { this(value, indexName, false); } + + public BoolItem(boolean value, String indexName, boolean isFromQuery) { + super(indexName, isFromQuery, new Substring(String.valueOf(value))); + this.value = value; + } + + /** Returns ItemType.WORD as we do not want a string binding from the parsed query to index types */ + @Override + public ItemType getItemType() { return ItemType.WORD; } + + @Override + public String getName() { return "BOOL"; } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); // takes care of index bytes + putString(stringValue(), buffer); + } + + public boolean value() { return value; } + + /** Returns "true" or "false" */ + @Override + public String stringValue() { return value ? "true" : "false"; } + + public void setValue(boolean value) { + this.value = value; + } + + /** + * Sets the value from a string + * + * @param stringValue "true" or "false" + * @throws IllegalArgumentException if the given value is not equal to "true" nor "false" (ignoring case) + */ + @Override + public void setValue(String stringValue) { + this.value = toBoolean(stringValue); + } + + private boolean toBoolean(String stringValue) { + switch (stringValue.toLowerCase()) { + case "true" : return true; + case "false" : return false; + default: throw new IllegalArgumentException("Expected 'true' or 'false', got '" + stringValue + "'"); + } + } + + /** Returns the same as stringValue */ + @Override + public String getRawWord() { + return stringValue(); + } + + @Override + public boolean isStemmed() { return false; } + + /** Word items uses a empty heading instead of "WORD " */ + @Override + protected void appendHeadingString(StringBuilder buffer) {} + + @Override + public int hashCode() { + return Boolean.hashCode(value); + } + + @Override + public boolean equals(Object object) { + if ( ! super.equals(object)) return false; + + BoolItem other = (BoolItem) object; // Ensured by superclass + return this.value == other.value; + } + + @Override + public int getNumWords() { return 1; } + + @Override + public String getIndexedString() { return stringValue(); } + + /** Returns true if this consists of regular word characters. Returns false if this represents a "special token" */ + @Override + public boolean isWords() { return false; } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java index 2e24937856c..714e8f9cb5e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java @@ -56,17 +56,6 @@ public class IntItem extends TermItem { expression = toExpression(from, to, 0); } - /** Returns the simplest expression matching this */ - private String toExpression(Limit from, Limit to, int hitLimit) { - if (from.equals(to) && hitLimit == 0) return from.number().toString(); - - String expression = from.toRangeStart() + ";" + to.toRangeEnd(); - if (hitLimit == 0) return expression; - - // Insert ;hitLimit at the end inside the brackets - return expression.substring(0, expression.length()-1) + ";" + hitLimit + expression.substring(expression.length()-1); - } - public IntItem(String expression) { this(expression, ""); } @@ -91,8 +80,19 @@ public class IntItem extends TermItem { this.expression = toExpression(from, to, hitLimit); } + /** Returns the simplest expression matching this */ + private String toExpression(Limit from, Limit to, int hitLimit) { + if (from.equals(to) && hitLimit == 0) return from.number().toString(); + + String expression = from.toRangeStart() + ";" + to.toRangeEnd(); + if (hitLimit == 0) return expression; + + // Insert ;hitLimit at the end inside the brackets + return expression.substring(0, expression.length()-1) + ";" + hitLimit + expression.substring(expression.length()-1); + } + /** Sets limit and flip them if "from" is greater than "to" */ - private final void setLimits(Limit from, Limit to) { + private void setLimits(Limit from, Limit to) { if (from.number().doubleValue() > to.number().doubleValue()) { this.from = to; this.to = from; @@ -188,8 +188,7 @@ public class IntItem extends TermItem { * <code>from</code> are returned, and if this number is negative the hits * closest to <code>to</code> are returned. * - * @param hitLimit - * number of hits to match for this operator + * @param hitLimit number of hits to match for this operator */ public final void setHitLimit(int hitLimit) { this.hitLimit = hitLimit; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java index 2535619f193..f230c76c954 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java @@ -54,17 +54,15 @@ public abstract class SimpleIndexedItem extends SimpleTaggableItem implements In } } + @Override public boolean equals(Object object) { - if (!super.equals(object)) { - return false; - } + if (!super.equals(object)) return false; + IndexedItem other = (IndexedItem) object; // Ensured by superclass - if (!this.index.equals(other.getIndexName())) { - return false; - } - return true; + return this.index.equals(other.getIndexName()); } + @Override public int hashCode() { return super.hashCode() + 113 * index.hashCode(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Substring.java b/container-search/src/main/java/com/yahoo/prelude/query/Substring.java index 10ea6f85b5f..ab3e28fc259 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Substring.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Substring.java @@ -19,14 +19,21 @@ public class Substring { /** The string this is a substring of */ public final String string; - public Substring(int start, int end,String string) { + /** Creates a substring which is identical to the string containing it */ + public Substring(String string) { + this.start = 0; + this.end = string.length(); + this.string=string; + } + + public Substring(int start, int end, String string) { this.start = start; this.end = end; this.string=string; } public String getValue() { - return string.substring(start,end); + return string.substring(start, end); } /** Returns the entire string this is a substring of. The start and end offsets are into this string. */ diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java index 764fb2b1118..8fdf147ce0b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java @@ -8,8 +8,7 @@ import java.nio.ByteBuffer; /** - * Superclass of "leaf" conditions containing a single entity which is either matched in - * a field or not. + * Superclass of "leaf" conditions containing a single entity which is either matched in a field or not. * * @author bratseth * @author havardpe @@ -116,8 +115,7 @@ public abstract class TermItem extends SimpleIndexedItem implements BlockItem { * but for historical reasons that is not the case. This method has nothing * to do with Unicode normalization. * - * @param normalizable - * set to true if accent removal can/should be performed + * @param normalizable set to true if accent removal can/should be performed */ public void setNormalizable(boolean normalizable) { this.normalizable = normalizable; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java index 0eb37373e1f..a0467703b13 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java @@ -6,7 +6,7 @@ package com.yahoo.prelude.query; * A term type enumeration * * @author bratseth - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class TermType { @@ -62,10 +62,9 @@ public class TermType { */ public Item createItemClass() { try { - return instanceClass.newInstance(); + return instanceClass.getDeclaredConstructor().newInstance(); } catch (Exception e) { - throw new RuntimeException("Could not create an instance for item " - + this, e); + throw new RuntimeException("Could not create an instance for item " + this, e); } } @@ -73,16 +72,15 @@ public class TermType { return sign; } + @Override public boolean equals(Object o) { - if (!(o instanceof TermType)) { - return false; - } + if ( ! (o instanceof TermType)) return false; TermType other = (TermType) o; - return name.equals(other.name); } + @Override public int hashCode() { return name.hashCode(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java index fb0a2418d37..72ba012ab07 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java @@ -6,12 +6,13 @@ import com.google.common.annotations.Beta; /** * Query tree helper methods and factories. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ @Beta public final class ToolBox { public static abstract class QueryVisitor { + /** * Called for each item in the query tree given to * {@link ToolBox#visit(QueryVisitor, Item)}. Return true to visit the @@ -29,6 +30,7 @@ public final class ToolBox { * visit() if there are no sub-items or visit() returned false. */ public abstract void onExit(); + } public static void visit(QueryVisitor visitor, Item item) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java index 39573e4d71f..be56c95a6af 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java @@ -33,14 +33,6 @@ public class WordItem extends TermItem { private boolean lowercased = false; - public ItemType getItemType() { - return ItemType.WORD; - } - - public String getName() { - return "WORD"; - } - public WordItem(String word) { this(word, ""); } @@ -70,12 +62,21 @@ public class WordItem extends TermItem { setWord(word); } + public ItemType getItemType() { + return ItemType.WORD; + } + + public String getName() { + return "WORD"; + } + public void setWord(String word) { Validator.ensureNotNull("The word of a word item", word); Validator.ensureNonEmpty("The word of a word item", word); this.word = word; } + @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); // takes care of index bytes putString(getEncodedWord(), buffer); @@ -112,6 +113,7 @@ public class WordItem extends TermItem { return word; } + @Override public boolean isStemmed() { return stemmed; } public void setStemmed(boolean stemmed) { this.stemmed = stemmed; } @@ -168,6 +170,7 @@ public class WordItem extends TermItem { } /** Returns true if this consists of regular word characters. Returns false if this represents a "special token" */ + @Override public boolean isWords() { return words; } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 1c2f409d792..36a04bb86e8 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -776,11 +776,11 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * different from default, while linguistics metadata are not added. * * @return a valid YQL+ query string or a human readable error message - * @see Query#yqlRepresentation(Tuple2, boolean) + * @see Query#yqlRepresentation(boolean) */ public String yqlRepresentation() { try { - return yqlRepresentation(null, true); + return yqlRepresentation(true); } catch (NullItemException e) { return "Query currently a placeholder, NullItem encountered."; } catch (RuntimeException e) { @@ -799,23 +799,22 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } } + /** @deprecated remove the ignored segmenterVersion argument from invocations */ + @Deprecated // TODO: Remove on Vespa 8 + public String yqlRepresentation(Tuple2<String, Version> segmenterVersion, boolean includeHitsAndOffset) { + return yqlRepresentation(includeHitsAndOffset); + } + /** * Serialize this query as YQL+. This will create a string representation * which should always be legal YQL+. If a problem occurs, a * RuntimeException is thrown. * - * @param segmenterVersion - * linguistics metadata used in federation, set to null if the - * annotation is not necessary - * @param includeHitsAndOffset - * whether to include hits and offset parameters converted to a - * offset/limit slice + * @param includeHitsAndOffset whether to include hits and offset parameters converted to a offset/limit slice * @return a valid YQL+ query string * @throws RuntimeException if there is a problem serializing the query tree */ - public String yqlRepresentation(@Nullable Tuple2<String, Version> segmenterVersion, boolean includeHitsAndOffset) { - String q = VespaSerializer.serialize(this); - + public String yqlRepresentation(boolean includeHitsAndOffset) { Set<String> sources = getModel().getSources(); Set<String> fields = getPresentation().getSummaryFields(); StringBuilder yql = new StringBuilder("select "); @@ -834,31 +833,20 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { commaSeparated(yql, sources); } yql.append(" where "); - if (segmenterVersion != null) { - yql.append("[{\"segmenter\": {\"version\": \"") - .append(segmenterVersion.second.toString()) - .append("\", \"backend\": \"") - .append(segmenterVersion.first).append("\"}}]("); - } - yql.append(q); - if (segmenterVersion != null) { - yql.append(')'); - } + yql.append(VespaSerializer.serialize(this)); if (getRanking().getSorting() != null && getRanking().getSorting().fieldOrders().size() > 0) { serializeSorting(yql); } if (includeHitsAndOffset) { if (getOffset() != 0) { - yql.append(" limit ") - .append(Integer.toString(getHits() + getOffset())) - .append(" offset ") - .append(Integer.toString(getOffset())); + yql.append(" limit ").append(getHits() + getOffset()) + .append(" offset ").append(getOffset()); } else if (getHits() != 10) { - yql.append(" limit ").append(Integer.toString(getHits())); + yql.append(" limit ").append(getHits()); } } if (getTimeout() != defaultTimeout) { - yql.append(" timeout ").append(Long.toString(getTimeout())); + yql.append(" timeout ").append(getTimeout()); } yql.append(';'); return yql.toString(); @@ -871,29 +859,41 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { if (yql.length() > initLen) { yql.append(", "); } - final Class<? extends AttributeSorter> sorterType = f.getSorter() - .getClass(); + Class<? extends AttributeSorter> sorterType = f.getSorter().getClass(); if (sorterType == Sorting.RawSorter.class) { - yql.append("[{\"").append(YqlParser.SORTING_FUNCTION) - .append("\": \"").append(Sorting.RAW).append("\"}]"); + yql.append("[{\"") + .append(YqlParser.SORTING_FUNCTION) + .append("\": \"") + .append(Sorting.RAW) + .append("\"}]"); } else if (sorterType == Sorting.LowerCaseSorter.class) { - yql.append("[{\"").append(YqlParser.SORTING_FUNCTION) - .append("\": \"").append(Sorting.LOWERCASE) - .append("\"}]"); + yql.append("[{\"") + .append(YqlParser.SORTING_FUNCTION) + .append("\": \"") + .append(Sorting.LOWERCASE) + .append("\"}]"); } else if (sorterType == Sorting.UcaSorter.class) { Sorting.UcaSorter uca = (Sorting.UcaSorter) f.getSorter(); String ucaLocale = uca.getLocale(); Sorting.UcaSorter.Strength ucaStrength = uca.getStrength(); - yql.append("[{\"").append(YqlParser.SORTING_FUNCTION) - .append("\": \"").append(Sorting.UCA).append("\""); + yql.append("[{\"") + .append(YqlParser.SORTING_FUNCTION) + .append("\": \"") + .append(Sorting.UCA) + .append("\""); if (ucaLocale != null) { - yql.append(", \"").append(YqlParser.SORTING_LOCALE) - .append("\": \"").append(ucaLocale).append('"'); + yql.append(", \"") + .append(YqlParser.SORTING_LOCALE) + .append("\": \"") + .append(ucaLocale) + .append('"'); } if (ucaStrength != Sorting.UcaSorter.Strength.UNDEFINED) { - yql.append(", \"").append(YqlParser.SORTING_STRENGTH) - .append("\": \"").append(ucaStrength.name()) - .append('"'); + yql.append(", \"") + .append(YqlParser.SORTING_STRENGTH) + .append("\": \"") + .append(ucaStrength.name()) + .append('"'); } yql.append("}]"); } @@ -906,8 +906,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Returns the context of this query, possibly creating it if missing. Returns the context, or null */ public QueryContext getContext(boolean create) { - if (context==null && create) - context=new QueryContext(getTraceLevel(),this); + if (context == null && create) + context = new QueryContext(getTraceLevel(),this); return context; } @@ -920,7 +920,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Returns whether the given query is equal to this */ @Override public boolean equals(Object other) { - if (this==other) return true; + if (this == other) return true; if ( ! (other instanceof Query)) return false; Query q = (Query) other; diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index db5b3e69803..10fa7d24d1a 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -5,13 +5,10 @@ import com.yahoo.collections.ArraySet; import com.yahoo.component.provider.ListenableFreezableClass; import com.yahoo.net.URI; import com.yahoo.prelude.hitfield.HitField; -import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.prelude.hitfield.XMLString; import com.yahoo.processing.Request; import com.yahoo.processing.response.Data; import com.yahoo.search.Query; import com.yahoo.search.Searcher; -import com.yahoo.text.XML; import java.util.Collections; import java.util.HashSet; @@ -22,7 +19,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.function.BiConsumer; -import java.util.stream.Collectors; /** * <p>A search hit. The identifier of the hit is the uri @@ -92,7 +88,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * used to order equal relevant hit by add order */ private int addNumber = -1; - private int sourceNumber; /** The query which produced this hit. Used for multi phase searching */ private Query query; diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index ac1c2ee4a6c..c27c047c899 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -61,6 +61,7 @@ import java.util.Map.Entry; import com.google.common.collect.ImmutableMap; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; +import com.yahoo.prelude.query.BoolItem; import com.yahoo.prelude.query.DotProductItem; import com.yahoo.prelude.query.EquivItem; import com.yahoo.prelude.query.ExactStringItem; @@ -102,9 +103,10 @@ import edu.umd.cs.findbugs.annotations.NonNull; /** * Serialize Vespa query trees to YQL+ strings. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class VespaSerializer { + // TODO refactor, too much copy/paste private static class AndSegmentSerializer extends Serializer { @@ -270,6 +272,7 @@ public class VespaSerializer { } private static class NotSerializer extends Serializer { + @Override void onExit(StringBuilder destination, Item item) { destination.append(')'); @@ -298,10 +301,8 @@ public class VespaSerializer { @Override boolean serialize(StringBuilder destination, Item item) { - throw new NullItemException( - "NullItem encountered in query tree." - + " This is usually a symptom of an invalid query or an error" - + " in a query transformer."); + throw new NullItemException("NullItem encountered in query tree. This is usually a symptom of an invalid " + + "query or an error in a query transformer."); } } @@ -402,6 +403,21 @@ public class VespaSerializer { } } + private static class BoolSerializer extends Serializer { + + @Override + void onExit(StringBuilder destination, Item item) { } + + @Override + boolean serialize(StringBuilder destination, Item item) { + BoolItem intItem = (BoolItem) item; + destination.append(normalizeIndexName(intItem.getIndexName())).append(" = "); + destination.append(((BoolItem) item).stringValue()); + return false; + } + + } + private static class RegExpSerializer extends Serializer { @Override @@ -419,6 +435,7 @@ public class VespaSerializer { } private static class ONearSerializer extends Serializer { + @Override void onExit(StringBuilder destination, Item item) { } @@ -452,6 +469,7 @@ public class VespaSerializer { } private static class OrSerializer extends Serializer { + @Override void onExit(StringBuilder destination, Item item) { destination.append(')'); @@ -1071,6 +1089,7 @@ public class VespaSerializer { dispatchBuilder.put(EquivItem.class, new EquivSerializer()); dispatchBuilder.put(ExactStringItem.class, new WordSerializer()); dispatchBuilder.put(IntItem.class, new NumberSerializer()); + dispatchBuilder.put(BoolItem.class, new BoolSerializer()); dispatchBuilder.put(MarkerWordItem.class, new WordSerializer()); // gotcha dispatchBuilder.put(NearItem.class, new NearSerializer()); dispatchBuilder.put(NotItem.class, new NotSerializer()); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 5928dd23b28..af095fefc1c 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -26,6 +26,7 @@ import com.yahoo.language.process.Segmenter; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; +import com.yahoo.prelude.query.BoolItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.DotProductItem; import com.yahoo.prelude.query.EquivItem; @@ -947,8 +948,10 @@ public class YqlParser implements Parser { String value = fetchConditionWord(ast); TermItem item; - if (value.equals("true") || value.equals("false")) - item = new WordItem(value, fetchConditionIndex(ast)); + if (value.equals("true")) { + item = new BoolItem(true, fetchConditionIndex(ast)); + } else if (value.equals("false")) + item = new BoolItem(false, fetchConditionIndex(ast)); else item = new IntItem(value, fetchConditionIndex(ast)); diff --git a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java index b23e25e173e..e89c8aeb409 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java @@ -318,12 +318,12 @@ public class MinimalQueryInserterTestCase { } @Test - public void testStringReprBasicSanity() { + public void testStringRepresentation() { String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203%3B"; Query query = new Query("search/?yql=" + yql); execution.search(query); - assertEquals("select ignoredfield from ignoredsource where [{\"segmenter\": {\"version\": \"1.9\", \"backend\": \"YqlUnitTest\"}}](title contains \"madonna\") order by something, shoesize desc limit 300 timeout 3;", - query.yqlRepresentation(new Tuple2<>("YqlUnitTest", new Version(1, 9)), true)); + assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by something, shoesize desc limit 300 timeout 3;", + query.yqlRepresentation()); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index 6984a8537ef..faf254577ce 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -94,7 +94,7 @@ public class VespaSerializerTestCase { } @Test - public final void testAnd() { + public void testAnd() { parseAndConfirm("(description contains \"a\" AND title contains \"that\")"); } @@ -103,39 +103,42 @@ public class VespaSerializerTestCase { } private void parseAndConfirm(String expected, String toParse) { - QueryTree item = parser - .parse(new Parsable() - .setQuery(SELECT + toParse + ";")); - // System.out.println(item.toString()); + QueryTree item = parser.parse(new Parsable().setQuery(SELECT + toParse + ";")); String q = VespaSerializer.serialize(item.getRoot()); assertEquals(expected, q); } @Test - public final void testAndNot() { + public void testAndNot() { parseAndConfirm("(description contains \"a\") AND !(title contains \"that\")"); } @Test - public final void testEquiv() { + public void testEquiv() { parseAndConfirm("title contains equiv(\"a\", \"b\")"); } @Test - public final void testNear() { + public void testNear() { parseAndConfirm("title contains near(\"a\", \"b\")"); parseAndConfirm("title contains ([{\"distance\": 50}]near(\"a\", \"b\"))"); } @Test - public final void testNumbers() { + public void testNumbers() { parseAndConfirm("title = 500"); parseAndConfirm("title > 500"); parseAndConfirm("title < 500"); } @Test - public final void testAnnotatedNumbers() { + public void testBoolean() { + parseAndConfirm("flag = true"); + parseAndConfirm("flag = false"); + } + + @Test + public void testAnnotatedNumbers() { parseAndConfirm("title = ([{\"filter\": true}]500)"); parseAndConfirm("title > ([{\"filter\": true}]500)"); parseAndConfirm("title < ([{\"filter\": true}](-500))"); @@ -144,32 +147,32 @@ public class VespaSerializerTestCase { } @Test - public final void testRange() { + public void testRange() { parseAndConfirm("range(title, 1, 500)"); } @Test - public final void testAnnotatedRange() { + public void testAnnotatedRange() { parseAndConfirm("[{\"filter\": true}]range(title, 1, 500)"); } @Test - public final void testOrderedNear() { + public void testOrderedNear() { parseAndConfirm("title contains onear(\"a\", \"b\")"); } @Test - public final void testOr() { + public void testOr() { parseAndConfirm("(description contains \"a\" OR title contains \"that\")"); } @Test - public final void testDotProduct() { + public void testDotProduct() { parseAndConfirm("dotProduct(description, {\"a\": 1, \"b\": 2})"); } @Test - public final void testPredicate() { + public void testPredicate() { parseAndConfirm("predicate(boolean,{\"gender\":\"male\"},{\"age\":25L})"); parseAndConfirm("predicate(boolean,{\"gender\":\"male\",\"hobby\":\"music\",\"hobby\":\"hiking\"}," + "{\"age\":25L})", @@ -182,32 +185,32 @@ public class VespaSerializerTestCase { } @Test - public final void testPhrase() { + public void testPhrase() { parseAndConfirm("description contains phrase(\"a\", \"b\")"); } @Test - public final void testAnnotatedPhrase() { + public void testAnnotatedPhrase() { parseAndConfirm("description contains ([{\"id\": 1}]phrase(\"a\", \"b\"))"); } @Test - public final void testAnnotatedNear() { + public void testAnnotatedNear() { parseAndConfirm("description contains ([{\"distance\": 37}]near(\"a\", \"b\"))"); } @Test - public final void testAnnotatedOnear() { + public void testAnnotatedOnear() { parseAndConfirm("description contains ([{\"distance\": 37}]onear(\"a\", \"b\"))"); } @Test - public final void testAnnotatedEquiv() { + public void testAnnotatedEquiv() { parseAndConfirm("description contains ([{\"id\": 1}]equiv(\"a\", \"b\"))"); } @Test - public final void testAnnotatedPhraseSegment() { + public void testAnnotatedPhraseSegment() { PhraseSegmentItem phraseSegment = new PhraseSegmentItem("abc", true, false); phraseSegment.addItem(new WordItem("a", "indexNamePlaceholder")); phraseSegment.addItem(new WordItem("b", "indexNamePlaceholder")); @@ -219,7 +222,7 @@ public class VespaSerializerTestCase { } @Test - public final void testSameElement() { + public void testSameElement() { SameElementItem sameElement = new SameElementItem("ss"); sameElement.addItem(new WordItem("a", "f1")); sameElement.addItem(new WordItem("b", "f2")); @@ -228,7 +231,7 @@ public class VespaSerializerTestCase { } @Test - public final void testAnnotatedAndSegment() { + public void testAnnotatedAndSegment() { AndSegmentItem andSegment = new AndSegmentItem("abc", true, false); andSegment.addItem(new WordItem("a", "indexNamePlaceholder")); andSegment.addItem(new WordItem("b", "indexNamePlaceholder")); @@ -239,32 +242,32 @@ public class VespaSerializerTestCase { } @Test - public final void testPhraseWithAnnotations() { + public void testPhraseWithAnnotations() { parseAndConfirm("description contains phrase(([{\"id\": 15}]\"a\"), \"b\")"); } @Test - public final void testPhraseSegmentInPhrase() { + public void testPhraseSegmentInPhrase() { parseAndConfirm("description contains phrase(\"a\", \"b\", ([{\"origin\": {\"original\": \"c d\", \"offset\": 0, \"length\": 3}}]phrase(\"c\", \"d\")))"); } @Test - public final void testRank() { + public void testRank() { parseAndConfirm("rank(a contains \"A\", b contains \"B\")"); } @Test - public final void testWand() { + public void testWand() { parseAndConfirm("wand(description, {\"a\": 1, \"b\": 2})"); } @Test - public final void testWeakAnd() { + public void testWeakAnd() { parseAndConfirm("weakAnd(a contains \"A\", b contains \"B\")"); } @Test - public final void testAnnotatedWeakAnd() { + public void testAnnotatedWeakAnd() { parseAndConfirm("([{\"" + YqlParser.TARGET_NUM_HITS + "\": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); parseAndConfirm("([{\"" + YqlParser.SCORE_THRESHOLD + "\": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); parseAndConfirm("([{\"" + YqlParser.TARGET_NUM_HITS + "\": 10, \"" + YqlParser.SCORE_THRESHOLD @@ -272,12 +275,12 @@ public class VespaSerializerTestCase { } @Test - public final void testWeightedSet() { + public void testWeightedSet() { parseAndConfirm("weightedSet(description, {\"a\": 1, \"b\": 2})"); } @Test - public final void testAnnotatedWord() { + public void testAnnotatedWord() { parseAndConfirm("description contains ([{\"andSegmenting\": true}]\"a\")"); parseAndConfirm("description contains ([{\"weight\": 37}]\"a\")"); parseAndConfirm("description contains ([{\"id\": 37}]\"a\")"); @@ -289,29 +292,29 @@ public class VespaSerializerTestCase { } @Test - public final void testPrefix() { + public void testPrefix() { parseAndConfirm("description contains ([{\"prefix\": true}]\"a\")"); } @Test - public final void testSuffix() { + public void testSuffix() { parseAndConfirm("description contains ([{\"suffix\": true}]\"a\")"); } @Test - public final void testSubstring() { + public void testSubstring() { parseAndConfirm("description contains ([{\"substring\": true}]\"a\")"); } @Test - public final void testExoticItemTypes() { + public void testExoticItemTypes() { Item item = MarkerWordItem.createEndOfHost(); String q = VespaSerializer.serialize(item); assertEquals("default contains ([{\"implicitTransforms\": false}]\"$\")", q); } @Test - public final void testEmptyIndex() { + public void testEmptyIndex() { Item item = new WordItem("nalle", true); String q = VespaSerializer.serialize(item); assertEquals("default contains \"nalle\"", q); @@ -319,7 +322,7 @@ public class VespaSerializerTestCase { @Test - public final void testLongAndNot() { + public void testLongAndNot() { NotItem item = new NotItem(); item.addItem(new WordItem("a")); item.addItem(new WordItem("b")); @@ -330,7 +333,7 @@ public class VespaSerializerTestCase { } @Test - public final void testPhraseAsOperatorArgument() { + public void testPhraseAsOperatorArgument() { // flattening phrases is a feature, not a bug parseAndConfirm("description contains phrase(\"a\", \"b\", \"c\")", "description contains phrase(\"a\", phrase(\"b\", \"c\"))"); @@ -344,7 +347,7 @@ public class VespaSerializerTestCase { } @Test - public final void testNumberTypeInt() { + public void testNumberTypeInt() { parseAndConfirm("title = 500"); parseAndConfirm("title > 500"); parseAndConfirm("title < (-500)"); @@ -354,7 +357,7 @@ public class VespaSerializerTestCase { } @Test - public final void testNumberTypeLong() { + public void testNumberTypeLong() { parseAndConfirm("title = 549755813888L"); parseAndConfirm("title > 549755813888L"); parseAndConfirm("title < (-549755813888L)"); @@ -364,7 +367,7 @@ public class VespaSerializerTestCase { } @Test - public final void testNumberTypeFloat() { + public void testNumberTypeFloat() { parseAndConfirm("title = 500.0"); // silly parseAndConfirm("title > 500.0"); parseAndConfirm("title < (-500.0)"); @@ -374,19 +377,19 @@ public class VespaSerializerTestCase { } @Test - public final void testAnnotatedLong() { + public void testAnnotatedLong() { parseAndConfirm("title >= ([{\"id\": 2014}](-549755813888L))"); } @Test - public final void testHitLimit() { + public void testHitLimit() { parseAndConfirm("title <= ([{\"hitLimit\": 89}](-500))"); parseAndConfirm("title <= ([{\"hitLimit\": 89}](-500))"); parseAndConfirm("[{\"hitLimit\": 89}]range(title, 1, 500)"); } @Test - public final void testOpenIntervals() { + public void testOpenIntervals() { parseAndConfirm("range(title, 0.0, 500.0)"); parseAndConfirm("[{\"bounds\": \"open\"}]range(title, 0.0, 500.0)"); parseAndConfirm("[{\"bounds\": \"leftOpen\"}]range(title, 0.0, 500.0)"); @@ -395,18 +398,18 @@ public class VespaSerializerTestCase { } @Test - public final void testRegExp() { + public void testRegExp() { parseAndConfirm("foo matches \"a b\""); } @Test - public final void testWordAlternatives() { + public void testWordAlternatives() { parseAndConfirm("foo contains" + " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + "alternatives({\"trees\": 1.0, \"tree\": 0.7}))"); } @Test - public final void testWordAlternativesInPhrase() { + public void testWordAlternativesInPhrase() { parseAndConfirm("foo contains phrase(\"forest\"," + " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + "alternatives({\"trees\": 1.0, \"tree\": 0.7}))" diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 31a057c158f..531167bb342 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -7,6 +7,7 @@ import com.yahoo.language.Language; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.BoolItem; import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.Item; @@ -249,7 +250,11 @@ public class YqlParserTestCase { @Test public void testBoolean() { assertParse("select foo from bar where flag = true;", "flag:true"); - assertParse("select foo from bar where flag = false;", "flag:false"); + QueryTree query = assertParse("select foo from bar where flag = false;", "flag:false"); + assertEquals(BoolItem.class, query.getRoot().getClass()); + BoolItem item = (BoolItem)query.getRoot(); + assertEquals("flag", item.getIndexName()); + assertEquals(false, item.value()); } @Test @@ -922,8 +927,10 @@ public class YqlParserTestCase { } } - private void assertParse(String yqlQuery, String expectedQueryTree) { - assertEquals(expectedQueryTree, parse(yqlQuery).toString()); + private QueryTree assertParse(String yqlQuery, String expectedQueryTree) { + QueryTree query = parse(yqlQuery); + assertEquals(expectedQueryTree, query.toString()); + return query; } private void assertCanonicalParse(String yqlQuery, String expectedQueryTree) { @@ -935,15 +942,17 @@ public class YqlParserTestCase { assertEquals(q.getModel().getQueryTree().toString(), expectedQueryTree); } - private void assertParseFail(String yqlQuery, Throwable expectedException) { + private QueryTree assertParseFail(String yqlQuery, Throwable expectedException) { + QueryTree query = null; try { - parse(yqlQuery); + query = parse(yqlQuery); } catch (Throwable t) { assertEquals(expectedException.getClass(), t.getClass()); assertEquals(expectedException.getMessage(), t.getMessage()); - return; + return query; } fail("Parse succeeded: " + yqlQuery); + return query; } private void assertSources(String yqlQuery, Collection<String> expectedSources) { diff --git a/dist/vespa.spec b/dist/vespa.spec index e520f282e1f..0a9e8a284ce 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -63,6 +63,7 @@ BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %endif +BuildRequires: xxhash-devel >= 0.6.5 BuildRequires: lz4-devel BuildRequires: libzstd-devel BuildRequires: zlib-devel @@ -97,6 +98,8 @@ Requires: perl-Pod-Usage Requires: perl-URI Requires: valgrind Requires: Judy +Requires: xxhash +Requires: xxhash-libs >= 0.6.5 Requires: lz4 Requires: libzstd Requires: zlib diff --git a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java index 779d706ac3c..824a3073515 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.json.readers; +import com.fasterxml.jackson.core.JsonToken; +import com.google.common.base.Preconditions; import com.yahoo.document.DataType; import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.FieldValue; @@ -14,6 +16,7 @@ import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue; public class ArrayReader { public static void fillArrayUpdate(TokenBuffer buffer, int initNesting, DataType valueType, List<FieldValue> arrayContents) { while (buffer.nesting() >= initNesting) { + Preconditions.checkArgument(buffer.currentToken() != JsonToken.VALUE_NULL, "Illegal null value for array entry"); arrayContents.add(readSingleValue(buffer, valueType)); buffer.next(); } @@ -25,6 +28,7 @@ public class ArrayReader { expectArrayStart(buffer.currentToken()); buffer.next(); while (buffer.nesting() >= initNesting) { + Preconditions.checkArgument(buffer.currentToken() != JsonToken.VALUE_NULL, "Illegal null value for array entry"); parent.add(readSingleValue(buffer, valueType)); buffer.next(); } diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java index 3d2d08842dc..cab55903b76 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.json.readers; +import com.fasterxml.jackson.core.JsonToken; import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StructuredFieldValue; @@ -18,8 +19,11 @@ public class StructReader { while (buffer.nesting() >= initNesting) { Field f = getField(buffer, parent); try { - FieldValue v = readSingleValue(buffer, f.getDataType()); - parent.setFieldValue(f, v); + // skip fields set to null + if (buffer.currentToken() != JsonToken.VALUE_NULL) { + FieldValue v = readSingleValue(buffer, f.getDataType()); + parent.setFieldValue(f, v); + } buffer.next(); } catch (IllegalArgumentException e) { throw new JsonReaderException(f, e); diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 9afebc16a3b..1ef671d90c0 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -167,6 +167,16 @@ public class JsonReaderTestCase { x.addField(new Field("integerfield", DataType.INT)); types.registerDocumentType(x); } + { + DocumentType x = new DocumentType("testnull"); + x.addField(new Field("intfield", DataType.INT)); + x.addField(new Field("stringfield", DataType.STRING)); + x.addField(new Field("arrayfield", new ArrayDataType(DataType.STRING))); + x.addField(new Field("weightedsetfield", new WeightedSetDataType(DataType.STRING, true, true))); + x.addField(new Field("mapfield", new MapDataType(DataType.STRING, DataType.STRING))); + x.addField(new Field("tensorfield", new TensorDataType(new TensorType.Builder().indexed("x").build()))); + types.registerDocumentType(x); + } } @After @@ -1166,6 +1176,48 @@ public class JsonReaderTestCase { } } + @Test + public void testNullValues() { + JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testnull::doc1',", + " 'fields': {", + " 'intfield': null,", + " 'stringfield': null,", + " 'arrayfield': null,", + " 'weightedsetfield': null,", + " 'mapfield': null,", + " 'tensorfield': null", + " }", + "}")); + DocumentPut put = (DocumentPut) r.readSingleDocument(DocumentParser.SupportedOperation.PUT, + "id:unittest:testnull::doc1"); + Document doc = put.getDocument(); + assertFieldValueNull(doc, "intfield"); + assertFieldValueNull(doc, "stringfield"); + assertFieldValueNull(doc, "arrayfield"); + assertFieldValueNull(doc, "weightedsetfield"); + assertFieldValueNull(doc, "mapfield"); + assertFieldValueNull(doc, "tensorfield"); + } + + @Test(expected=JsonReaderException.class) + public void testNullArrayElement() { + JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testnull::doc1',", + " 'fields': {", + " 'arrayfield': [ null ]", + " }", + "}")); + r.readSingleDocument(DocumentParser.SupportedOperation.PUT, "id:unittest:testnull::doc1"); + fail(); + } + + private void assertFieldValueNull(Document doc, String fieldName) { + Field field = doc.getField(fieldName); + assertNotNull(field); + FieldValue fieldValue = doc.getFieldValue(field); + assertNull(fieldValue); + } + + static ByteArrayInputStream jsonToInputStream(String json) { return new ByteArrayInputStream(Utf8.toBytes(json)); } diff --git a/document/src/vespa/document/annotation/annotation.h b/document/src/vespa/document/annotation/annotation.h index f0cb8088b1f..94ffe12eb85 100644 --- a/document/src/vespa/document/annotation/annotation.h +++ b/document/src/vespa/document/annotation/annotation.h @@ -7,7 +7,7 @@ #include <memory> namespace document { -class SpanNode; +struct SpanNode; class Annotation { const AnnotationType * _type; diff --git a/document/src/vespa/document/annotation/spannode.h b/document/src/vespa/document/annotation/spannode.h index 742b1534a33..0a5bbdcaab8 100644 --- a/document/src/vespa/document/annotation/spannode.h +++ b/document/src/vespa/document/annotation/spannode.h @@ -5,7 +5,7 @@ #include <vespa/vespalib/stllike/string.h> namespace document { -class SpanTreeVisitor; +struct SpanTreeVisitor; struct SpanNode { typedef std::unique_ptr<SpanNode> UP; diff --git a/document/src/vespa/document/annotation/spantree.h b/document/src/vespa/document/annotation/spantree.h index 4acd39b4793..d181d755fae 100644 --- a/document/src/vespa/document/annotation/spantree.h +++ b/document/src/vespa/document/annotation/spantree.h @@ -7,8 +7,8 @@ #include <cassert> namespace document { -class SpanNode; -class SpanTreeVisitor; +struct SpanNode; +struct SpanTreeVisitor; class SpanTree { typedef std::vector<Annotation> AnnotationVector; diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index 8025365be26..0a288aeb69d 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -17,7 +17,7 @@ namespace document { class Document; class DocumentType; class DocumentTypeRepo; -class FieldValueWriter; +struct FieldValueWriter; class FixedTypeRepo; class FieldSet; class StructDataType; @@ -123,8 +123,8 @@ private: VESPA_DLL_LOCAL const StructDataType & getStructType() const; // Iterator implementation - class FieldIterator; - friend class FieldIterator; + struct FieldIterator; + friend struct FieldIterator; StructuredIterator::UP getIterator(const Field* toFind) const override; diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.h b/document/src/vespa/document/fieldvalue/tensorfieldvalue.h index 5c831601d1a..49088c587c6 100644 --- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.h @@ -4,7 +4,7 @@ #include "fieldvalue.h" -namespace vespalib { namespace tensor { class Tensor; } } +namespace vespalib { namespace tensor { struct Tensor; } } namespace document { diff --git a/document/src/vespa/document/predicate/predicate.h b/document/src/vespa/document/predicate/predicate.h index 97823a3ad2b..4c6351f5e4c 100644 --- a/document/src/vespa/document/predicate/predicate.h +++ b/document/src/vespa/document/predicate/predicate.h @@ -8,7 +8,7 @@ namespace vespalib { class Slime; -namespace slime { class Inspector; } +namespace slime { struct Inspector; } } // namespace vespalib namespace document { diff --git a/document/src/vespa/document/predicate/predicate_builder.h b/document/src/vespa/document/predicate/predicate_builder.h index 14cb9eb6496..3750e3aa615 100644 --- a/document/src/vespa/document/predicate/predicate_builder.h +++ b/document/src/vespa/document/predicate/predicate_builder.h @@ -7,7 +7,7 @@ #include <vector> namespace document { -class PredicateNode; +struct PredicateNode; class PredicateBuilder : private PredicateSlimeVisitor { std::vector<PredicateNode *>_nodes; diff --git a/document/src/vespa/document/predicate/predicate_slime_visitor.h b/document/src/vespa/document/predicate/predicate_slime_visitor.h index 5727a4d4269..394c72fe8dd 100644 --- a/document/src/vespa/document/predicate/predicate_slime_visitor.h +++ b/document/src/vespa/document/predicate/predicate_slime_visitor.h @@ -3,7 +3,7 @@ #pragma once namespace vespalib { -namespace slime { class Inspector; } +namespace slime { struct Inspector; } } // namespace vespalib namespace document { diff --git a/document/src/vespa/document/repo/configbuilder.h b/document/src/vespa/document/repo/configbuilder.h index c389fd3b09e..856ffaeda08 100644 --- a/document/src/vespa/document/repo/configbuilder.h +++ b/document/src/vespa/document/repo/configbuilder.h @@ -11,7 +11,7 @@ namespace document::config_builder { -class TypeOrId; +struct TypeOrId; struct DatatypeConfig : DocumenttypesConfig::Documenttype::Datatype { static int32_t id_counter; diff --git a/document/src/vespa/document/repo/documenttyperepo.h b/document/src/vespa/document/repo/documenttyperepo.h index 68e4087ca49..f0c59918a74 100644 --- a/document/src/vespa/document/repo/documenttyperepo.h +++ b/document/src/vespa/document/repo/documenttyperepo.h @@ -15,7 +15,7 @@ namespace internal { class AnnotationType; class DataType; -class DataTypeRepo; +struct DataTypeRepo; class DocumentType; class DocumentTypeRepo { diff --git a/document/src/vespa/document/serialization/annotationserializer.h b/document/src/vespa/document/serialization/annotationserializer.h index a84087ce6d1..d8cdd833939 100644 --- a/document/src/vespa/document/serialization/annotationserializer.h +++ b/document/src/vespa/document/serialization/annotationserializer.h @@ -13,7 +13,7 @@ class Annotation; class Span; class SpanList; class SimpleSpanList; -class SpanNode; +struct SpanNode; class SpanTree; class AnnotationSerializer : private SpanTreeVisitor { diff --git a/eval/src/vespa/eval/eval/basic_nodes.h b/eval/src/vespa/eval/eval/basic_nodes.h index 95e6cdf34f7..6af0a04b31a 100644 --- a/eval/src/vespa/eval/eval/basic_nodes.h +++ b/eval/src/vespa/eval/eval/basic_nodes.h @@ -14,7 +14,7 @@ namespace vespalib::eval { -namespace nodes { class Node; } +namespace nodes { struct Node; } struct NodeTraverser; struct NodeVisitor; diff --git a/eval/src/vespa/eval/eval/compile_tensor_function.h b/eval/src/vespa/eval/eval/compile_tensor_function.h index bfac0e0f036..013d228c2f9 100644 --- a/eval/src/vespa/eval/eval/compile_tensor_function.h +++ b/eval/src/vespa/eval/eval/compile_tensor_function.h @@ -9,7 +9,7 @@ namespace vespalib { class Stash; } namespace vespalib::eval { -class TensorFunction; +struct TensorFunction; std::vector<InterpretedFunction::Instruction> compile_tensor_function(const TensorFunction &function, Stash &stash); diff --git a/eval/src/vespa/eval/eval/gbdt.h b/eval/src/vespa/eval/eval/gbdt.h index 23d937604ca..378b6cc4b08 100644 --- a/eval/src/vespa/eval/eval/gbdt.h +++ b/eval/src/vespa/eval/eval/gbdt.h @@ -8,7 +8,7 @@ namespace vespalib { namespace eval { -namespace nodes { class Node; } +namespace nodes { struct Node; } namespace gbdt { diff --git a/eval/src/vespa/eval/eval/interpreted_function.h b/eval/src/vespa/eval/eval/interpreted_function.h index a9b6082fa4e..dbe5345fc9b 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.h +++ b/eval/src/vespa/eval/eval/interpreted_function.h @@ -10,9 +10,9 @@ namespace vespalib { namespace eval { -namespace nodes { class Node; } -class TensorEngine; -class TensorFunction; +namespace nodes { struct Node; } +struct TensorEngine; +struct TensorFunction; class TensorSpec; /** diff --git a/eval/src/vespa/eval/eval/make_tensor_function.h b/eval/src/vespa/eval/eval/make_tensor_function.h index f9f415b9b9f..bf6a9a44f4a 100644 --- a/eval/src/vespa/eval/eval/make_tensor_function.h +++ b/eval/src/vespa/eval/eval/make_tensor_function.h @@ -6,11 +6,11 @@ namespace vespalib { class Stash; } namespace vespalib::eval { -class TensorEngine; +struct TensorEngine; class NodeTypes; -class TensorFunction; +struct TensorFunction; -namespace nodes { class Node; } +namespace nodes { struct Node; } const TensorFunction &make_tensor_function(const TensorEngine &engine, const nodes::Node &root, const NodeTypes &types, Stash &stash); diff --git a/eval/src/vespa/eval/eval/node_types.h b/eval/src/vespa/eval/eval/node_types.h index f8a13bfb2ac..0b049598f4c 100644 --- a/eval/src/vespa/eval/eval/node_types.h +++ b/eval/src/vespa/eval/eval/node_types.h @@ -8,7 +8,7 @@ namespace vespalib { namespace eval { -namespace nodes { class Node; } +namespace nodes { struct Node; } class Function; /** diff --git a/eval/src/vespa/eval/eval/tensor.h b/eval/src/vespa/eval/eval/tensor.h index 149e2774bfb..ddc341ed910 100644 --- a/eval/src/vespa/eval/eval/tensor.h +++ b/eval/src/vespa/eval/eval/tensor.h @@ -8,7 +8,7 @@ namespace vespalib { namespace eval { -class TensorEngine; +struct TensorEngine; /** * Base class for all tensors. Tensor operations are defined by the diff --git a/eval/src/vespa/eval/eval/tensor_engine.h b/eval/src/vespa/eval/eval/tensor_engine.h index a01a6f889fd..d9f6d35609a 100644 --- a/eval/src/vespa/eval/eval/tensor_engine.h +++ b/eval/src/vespa/eval/eval/tensor_engine.h @@ -17,7 +17,7 @@ class nbostream; namespace eval { -class Value; +struct Value; class Tensor; class TensorSpec; diff --git a/eval/src/vespa/eval/eval/tensor_spec.h b/eval/src/vespa/eval/eval/tensor_spec.h index 66feb02de40..25af4c7a93c 100644 --- a/eval/src/vespa/eval/eval/tensor_spec.h +++ b/eval/src/vespa/eval/eval/tensor_spec.h @@ -11,8 +11,8 @@ namespace vespalib { namespace slime { -class Cursor; -class Inspector; +struct Cursor; +struct Inspector; } // namespace vespalib::slime diff --git a/eval/src/vespa/eval/eval/visit_stuff.h b/eval/src/vespa/eval/eval/visit_stuff.h index cbbca9be056..ef21352535d 100644 --- a/eval/src/vespa/eval/eval/visit_stuff.h +++ b/eval/src/vespa/eval/eval/visit_stuff.h @@ -9,7 +9,7 @@ namespace vespalib { class ObjectVisitor; namespace eval { enum class Aggr; -class TensorFunction; +struct TensorFunction; namespace visit { using map_fun_t = double (*)(double); using join_fun_t = double (*)(double, double); diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.h index 49e075f6999..52d49b8e95b 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.h @@ -3,7 +3,7 @@ #pragma once namespace vespalib::tensor { - class Tensor; + struct Tensor; class DenseTensor; } diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h index e80cd063e30..b53a3af9e19 100644 --- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h @@ -8,11 +8,11 @@ namespace vespalib { class Slime; -namespace slime { class Inserter; } +namespace slime { struct Inserter; } namespace tensor { -class Tensor; +struct Tensor; class TensorBuilder; /** diff --git a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h index db05574dfce..23d77a5c704 100644 --- a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h @@ -8,7 +8,7 @@ class nbostream; namespace tensor { -class Tensor; +struct Tensor; class TensorBuilder; /** diff --git a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.h b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.h index c655210907f..38c3373213c 100644 --- a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.h @@ -11,7 +11,7 @@ class nbostream; namespace tensor { -class Tensor; +struct Tensor; class TensorBuilder; /** diff --git a/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt b/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt index db31b08e3e4..57855dc7eaa 100644 --- a/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(eval_tensor_sparse OBJECT sparse_tensor_address_combiner.cpp sparse_tensor_address_padder.cpp sparse_tensor_address_reducer.cpp + sparse_tensor_address_ref.cpp sparse_tensor_match.cpp sparse_tensor_modify.cpp sparse_tensor_builder.cpp diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.cpp new file mode 100644 index 00000000000..ba40e37488e --- /dev/null +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.cpp @@ -0,0 +1,14 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "sparse_tensor_address_ref.h" +#include <xxhash.h> + +namespace vespalib::tensor { + +uint32_t +SparseTensorAddressRef::calcHash() const { + return XXH32(_start, _size, 0); +} + +} + diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h index 8533aa29829..743a46d63e1 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/vespalib/util/stash.h> -#include <vespa/vespalib/xxhash/xxhash.h> #include <cstring> namespace vespalib::tensor { @@ -41,7 +40,7 @@ public: uint32_t hash() const { return _hash; } - uint32_t calcHash() const { return XXH32(_start, _size, 0); } + uint32_t calcHash() const; bool operator<(const SparseTensorAddressRef &rhs) const { size_t minSize = std::min(_size, rhs._size); diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.h index ec6edf2d847..2c1c57de8ac 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.h @@ -3,7 +3,7 @@ #pragma once namespace vespalib::tensor { - class Tensor; + struct Tensor; class SparseTensor; } diff --git a/eval/src/vespa/eval/tensor/tensor.h b/eval/src/vespa/eval/tensor/tensor.h index 842fed436cd..8e10af04b18 100644 --- a/eval/src/vespa/eval/tensor/tensor.h +++ b/eval/src/vespa/eval/tensor/tensor.h @@ -10,7 +10,7 @@ #include <vespa/eval/eval/value_type.h> namespace vespalib { -namespace eval { class BinaryOperation; } +namespace eval { struct BinaryOperation; } namespace tensor { class TensorVisitor; diff --git a/eval/src/vespa/eval/tensor/tensor_builder.h b/eval/src/vespa/eval/tensor/tensor_builder.h index 05238b27df5..205da137895 100644 --- a/eval/src/vespa/eval/tensor/tensor_builder.h +++ b/eval/src/vespa/eval/tensor/tensor_builder.h @@ -7,7 +7,7 @@ namespace vespalib { namespace tensor { -class Tensor; +struct Tensor; /** * An interfrace for builder of tensors (sparse multi-dimensional array). diff --git a/eval/src/vespa/eval/tensor/tensor_factory.h b/eval/src/vespa/eval/tensor/tensor_factory.h index 5fe31afc4dd..f3c9aae7328 100644 --- a/eval/src/vespa/eval/tensor/tensor_factory.h +++ b/eval/src/vespa/eval/tensor/tensor_factory.h @@ -8,7 +8,7 @@ namespace vespalib { namespace tensor { -class Tensor; +struct Tensor; class TensorBuilder; /** diff --git a/eval/src/vespa/eval/tensor/tensor_mapper.h b/eval/src/vespa/eval/tensor/tensor_mapper.h index 99994bd15e8..6218146496a 100644 --- a/eval/src/vespa/eval/tensor/tensor_mapper.h +++ b/eval/src/vespa/eval/tensor/tensor_mapper.h @@ -7,7 +7,7 @@ namespace vespalib { namespace tensor { -class Tensor; +struct Tensor; /** * Class to map a tensor to a given tensor type. Dimensions in input diff --git a/fastos/src/vespa/fastos/unix_ipc.cpp b/fastos/src/vespa/fastos/unix_ipc.cpp index 79fbe3ee076..d1b7d199782 100644 --- a/fastos/src/vespa/fastos/unix_ipc.cpp +++ b/fastos/src/vespa/fastos/unix_ipc.cpp @@ -660,7 +660,7 @@ void FastOS_UNIX_IPCHelper::DeliverMessages (FastOS_RingBuffer *buffer) if((readSpace - sizeof(int)) >= bufferData->_messageSize) { - _app->OnReceivedIPCMessage(&bufferData->_buffer[sizeof(int)], + _app->OnReceivedIPCMessage(&bufferData->_buffer[0] + sizeof(int), bufferData->_messageSize); buffer->Consume(sizeof(int) + bufferData->_messageSize); buffer->RepositionDataAt0(); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 910dbbe6249..1f260e9d158 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -12,6 +12,7 @@ import java.util.TreeMap; import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; +import static com.yahoo.vespa.flags.FetchVector.Dimension.ZONE_ID; /** * @author hakonhall @@ -89,6 +90,12 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); + public static final UnboundBooleanFlag USE_ADAPTIVE_DISPATCH = defineFeatureFlag( + "use-adaptive-dispatch", false, + "Should adaptive dispatch be used over round robin", + "Takes effect at redeployment", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/fnet/src/tests/examples/examples_test.cpp b/fnet/src/tests/examples/examples_test.cpp index c823c52b995..61a2408e39d 100644 --- a/fnet/src/tests/examples/examples_test.cpp +++ b/fnet/src/tests/examples/examples_test.cpp @@ -9,14 +9,6 @@ // reserved in vespa/factory/doc/port-ranges.txt static const int PORT0 = 18570; static const int PORT1 = 18571; -static const int PORT2 = 18572; -static const int PORT3 = 18573; -static const int PORT4 = 18574; -static const int PORT5 = 18575; -static const int PORT6 = 18576; -static const int PORT7 = 18577; -static const int PORT8 = 18578; -static const int PORT9 = 18579; using vespalib::SlaveProc; diff --git a/fnet/src/tests/locking/lockspeed.cpp b/fnet/src/tests/locking/lockspeed.cpp index aa4f6649fd9..50c8d4eef1b 100644 --- a/fnet/src/tests/locking/lockspeed.cpp +++ b/fnet/src/tests/locking/lockspeed.cpp @@ -144,16 +144,16 @@ TEST("lock speed") { start.SetNow(); for (i = 0; i < 1000000; i++) { - std::mutex lock0; - std::mutex lock1; - std::mutex lock2; - std::mutex lock3; - std::mutex lock4; - std::mutex lock5; - std::mutex lock6; - std::mutex lock7; - std::mutex lock8; - std::mutex lock9; + [[maybe_unused]] std::mutex lock0; + [[maybe_unused]] std::mutex lock1; + [[maybe_unused]] std::mutex lock2; + [[maybe_unused]] std::mutex lock3; + [[maybe_unused]] std::mutex lock4; + [[maybe_unused]] std::mutex lock5; + [[maybe_unused]] std::mutex lock6; + [[maybe_unused]] std::mutex lock7; + [[maybe_unused]] std::mutex lock8; + [[maybe_unused]] std::mutex lock9; } stop.SetNow(); stop -= start; diff --git a/fnet/src/vespa/fnet/channellookup.h b/fnet/src/vespa/fnet/channellookup.h index 52365a89e5f..63a160a9955 100644 --- a/fnet/src/vespa/fnet/channellookup.h +++ b/fnet/src/vespa/fnet/channellookup.h @@ -6,7 +6,7 @@ #include <vector> namespace fnet { - class ChannelMap; + struct ChannelMap; } class FNET_Channel; diff --git a/fnet/src/vespa/fnet/frt/reflection.h b/fnet/src/vespa/fnet/frt/reflection.h index 5189cf81d0a..3aee3267b12 100644 --- a/fnet/src/vespa/fnet/frt/reflection.h +++ b/fnet/src/vespa/fnet/frt/reflection.h @@ -7,7 +7,7 @@ class FRT_Values; class FRT_Supervisor; -class FRT_StringValue; +struct FRT_StringValue; class FRT_Method { diff --git a/fnet/src/vespa/fnet/frt/values.cpp b/fnet/src/vespa/fnet/frt/values.cpp index 75661b4983b..a5f59df19b2 100644 --- a/fnet/src/vespa/fnet/frt/values.cpp +++ b/fnet/src/vespa/fnet/frt/values.cpp @@ -129,7 +129,7 @@ FRT_Values::EnsureFree(uint32_t need) char *types = (char *) _stash.alloc(cnt + 1); memcpy(types, _typeString, _numValues); memset(types + _numValues, FRT_VALUE_NONE, cnt + 1 - _numValues); - FRT_Value *values = (FRT_Value *) _stash.alloc(cnt * sizeof(FRT_Value)); + FRT_Value *values = (FRT_Value *) (void *)_stash.alloc(cnt * sizeof(FRT_Value)); memcpy(values, _values, _numValues * sizeof(FRT_Value)); _maxValues = cnt; _typeString = types; @@ -159,7 +159,7 @@ FRT_Values::AddInt8Array(const uint8_t *array, uint32_t len) { uint16_t * FRT_Values::AddInt16Array(uint32_t len) { EnsureFree(); - uint16_t *ret = (uint16_t *) _stash.alloc(len * sizeof(uint16_t)); + uint16_t *ret = (uint16_t *)(void *) _stash.alloc(len * sizeof(uint16_t)); _values[_numValues]._int16_array._pt = ret; _values[_numValues]._int16_array._len = len; _typeString[_numValues++] = FRT_VALUE_INT16_ARRAY; @@ -169,7 +169,7 @@ FRT_Values::AddInt16Array(uint32_t len) { void FRT_Values::AddInt16Array(const uint16_t *array, uint32_t len) { EnsureFree(); - uint16_t *pt = (uint16_t *) _stash.alloc(len * sizeof(uint16_t)); + uint16_t *pt = (uint16_t *)(void *) _stash.alloc(len * sizeof(uint16_t)); _values[_numValues]._int16_array._pt = pt; _values[_numValues]._int16_array._len = len; _typeString[_numValues++] = FRT_VALUE_INT16_ARRAY; @@ -179,7 +179,7 @@ FRT_Values::AddInt16Array(const uint16_t *array, uint32_t len) { uint32_t * FRT_Values::AddInt32Array(uint32_t len) { EnsureFree(); - uint32_t *ret = (uint32_t *) _stash.alloc(len * sizeof(uint32_t)); + uint32_t *ret = (uint32_t *)(void *) _stash.alloc(len * sizeof(uint32_t)); _values[_numValues]._int32_array._pt = ret; _values[_numValues]._int32_array._len = len; _typeString[_numValues++] = FRT_VALUE_INT32_ARRAY; @@ -189,7 +189,7 @@ FRT_Values::AddInt32Array(uint32_t len) { void FRT_Values::AddInt32Array(const uint32_t *array, uint32_t len) { EnsureFree(); - uint32_t *pt = (uint32_t *) _stash.alloc(len * sizeof(uint32_t)); + uint32_t *pt = (uint32_t *)(void *) _stash.alloc(len * sizeof(uint32_t)); _values[_numValues]._int32_array._pt = pt; _values[_numValues]._int32_array._len = len; _typeString[_numValues++] = FRT_VALUE_INT32_ARRAY; @@ -199,7 +199,7 @@ FRT_Values::AddInt32Array(const uint32_t *array, uint32_t len) { uint64_t * FRT_Values::AddInt64Array(uint32_t len) { EnsureFree(); - uint64_t *ret = (uint64_t *) _stash.alloc(len * sizeof(uint64_t)); + uint64_t *ret = (uint64_t *)(void *) _stash.alloc(len * sizeof(uint64_t)); _values[_numValues]._int64_array._pt = ret; _values[_numValues]._int64_array._len = len; _typeString[_numValues++] = FRT_VALUE_INT64_ARRAY; @@ -209,7 +209,7 @@ FRT_Values::AddInt64Array(uint32_t len) { void FRT_Values::AddInt64Array(const uint64_t *array, uint32_t len) { EnsureFree(); - uint64_t *pt = (uint64_t *) _stash.alloc(len * sizeof(uint64_t)); + uint64_t *pt = (uint64_t *)(void *) _stash.alloc(len * sizeof(uint64_t)); _values[_numValues]._int64_array._pt = pt; _values[_numValues]._int64_array._len = len; _typeString[_numValues++] = FRT_VALUE_INT64_ARRAY; @@ -219,7 +219,7 @@ FRT_Values::AddInt64Array(const uint64_t *array, uint32_t len) { float * FRT_Values::AddFloatArray(uint32_t len) { EnsureFree(); - float *ret = (float *) _stash.alloc(len * sizeof(float)); + float *ret = (float *)(void *) _stash.alloc(len * sizeof(float)); _values[_numValues]._float_array._pt = ret; _values[_numValues]._float_array._len = len; _typeString[_numValues++] = FRT_VALUE_FLOAT_ARRAY; @@ -229,7 +229,7 @@ FRT_Values::AddFloatArray(uint32_t len) { void FRT_Values::AddFloatArray(const float *array, uint32_t len) { EnsureFree(); - float *pt = (float *) _stash.alloc(len * sizeof(float)); + float *pt = (float *)(void *) _stash.alloc(len * sizeof(float)); _values[_numValues]._float_array._pt = pt; _values[_numValues]._float_array._len = len; _typeString[_numValues++] = FRT_VALUE_FLOAT_ARRAY; @@ -239,7 +239,7 @@ FRT_Values::AddFloatArray(const float *array, uint32_t len) { double * FRT_Values::AddDoubleArray(uint32_t len) { EnsureFree(); - double *ret = (double *) _stash.alloc(len * sizeof(double)); + double *ret = (double *)(void *) _stash.alloc(len * sizeof(double)); _values[_numValues]._double_array._pt = ret; _values[_numValues]._double_array._len = len; _typeString[_numValues++] = FRT_VALUE_DOUBLE_ARRAY; @@ -249,7 +249,7 @@ FRT_Values::AddDoubleArray(uint32_t len) { void FRT_Values::AddDoubleArray(const double *array, uint32_t len) { EnsureFree(); - double *pt = (double *) _stash.alloc(len * sizeof(double)); + double *pt = (double *)(void *) _stash.alloc(len * sizeof(double)); _values[_numValues]._double_array._pt = pt; _values[_numValues]._double_array._len = len; _typeString[_numValues++] = FRT_VALUE_DOUBLE_ARRAY; @@ -277,7 +277,7 @@ FRT_Values::AddString(uint32_t len) { FRT_StringValue * FRT_Values::AddStringArray(uint32_t len) { EnsureFree(); - FRT_StringValue *ret = (FRT_StringValue *) _stash.alloc(len * sizeof(FRT_StringValue)); + FRT_StringValue *ret = (FRT_StringValue *)(void *) _stash.alloc(len * sizeof(FRT_StringValue)); _values[_numValues]._string_array._pt = ret; _values[_numValues]._string_array._len = len; _typeString[_numValues++] = FRT_VALUE_STRING_ARRAY; @@ -327,7 +327,7 @@ FRT_Values::AddData(uint32_t len) { FRT_DataValue * FRT_Values::AddDataArray(uint32_t len) { EnsureFree(); - FRT_DataValue *ret = (FRT_DataValue *) _stash.alloc(len * sizeof(FRT_DataValue)); + FRT_DataValue *ret = (FRT_DataValue *)(void *) _stash.alloc(len * sizeof(FRT_DataValue)); _values[_numValues]._data_array._pt = ret; _values[_numValues]._data_array._len = len; _typeString[_numValues++] = FRT_VALUE_DATA_ARRAY; diff --git a/fnet/src/vespa/fnet/frt/values.h b/fnet/src/vespa/fnet/frt/values.h index 0b81a9f9f47..e00aec8423c 100644 --- a/fnet/src/vespa/fnet/frt/values.h +++ b/fnet/src/vespa/fnet/frt/values.h @@ -9,7 +9,7 @@ namespace fnet { char * copyString(char *dst, const char *src, size_t len); char * copyData(char *dst, const void *src, size_t len); - class BlobRef; + struct BlobRef; } class FNET_DataBuffer; diff --git a/fnet/src/vespa/fnet/transport.cpp b/fnet/src/vespa/fnet/transport.cpp index 80dfd118c20..fdc3576db2f 100644 --- a/fnet/src/vespa/fnet/transport.cpp +++ b/fnet/src/vespa/fnet/transport.cpp @@ -4,7 +4,7 @@ #include "transport_thread.h" #include "iocomponent.h" #include <chrono> -#include <vespa/vespalib/xxhash/xxhash.h> +#include <xxhash.h> namespace { diff --git a/fsa/src/vespa/fsamanagers/mutex.h b/fsa/src/vespa/fsamanagers/mutex.h index 05ea2adc732..93f17e2e757 100644 --- a/fsa/src/vespa/fsamanagers/mutex.h +++ b/fsa/src/vespa/fsamanagers/mutex.h @@ -23,7 +23,7 @@ namespace fsa { class Mutex { protected: - class Impl; + struct Impl; Impl *_impl; public: diff --git a/fsa/src/vespa/fsamanagers/rwlock.h b/fsa/src/vespa/fsamanagers/rwlock.h index 395cf66c468..f441ad15ff4 100644 --- a/fsa/src/vespa/fsamanagers/rwlock.h +++ b/fsa/src/vespa/fsamanagers/rwlock.h @@ -23,7 +23,7 @@ namespace fsa { class RWLock { protected: - class Impl; + struct Impl; Impl *_impl; public: diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java index 1291418083b..770c59dec35 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.core; -import com.yahoo.yolean.system.CatchSigTerm; +import com.yahoo.yolean.system.CatchSignals; import java.util.Arrays; import java.util.concurrent.atomic.AtomicBoolean; @@ -41,7 +41,7 @@ public class StandaloneMain { System.out.println("debug\tInitializing application without privileges."); loader.init(bundleLocation, false); loader.start(); - setupSigTermHandler(); + setupSignalHandlers(); waitForShutdown(); System.out.println("debug\tTrying to shutdown in a controlled manner."); log.log(Level.INFO, "JDisc shutting down"); @@ -59,8 +59,8 @@ public class StandaloneMain { } private final AtomicBoolean signalCaught = new AtomicBoolean(false); - private void setupSigTermHandler() { - CatchSigTerm.setup(signalCaught); // catch termination signal + private void setupSignalHandlers() { + CatchSignals.setup(signalCaught); } private void waitForShutdown() { synchronized (signalCaught) { diff --git a/logserver/pom.xml b/logserver/pom.xml index be2d8ba4cee..1045b93185e 100644 --- a/logserver/pom.xml +++ b/logserver/pom.xml @@ -26,6 +26,11 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>yolean</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>vespalog</artifactId> <version>${project.version}</version> </dependency> diff --git a/logserver/src/main/java/com/yahoo/logserver/Server.java b/logserver/src/main/java/com/yahoo/logserver/Server.java index 36b1c3e31a0..d0df5bf2972 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Server.java +++ b/logserver/src/main/java/com/yahoo/logserver/Server.java @@ -10,7 +10,7 @@ import com.yahoo.logserver.handlers.HandlerThread; import com.yahoo.logserver.handlers.LogHandler; import com.yahoo.logserver.net.LogConnectionFactory; import com.yahoo.logserver.net.control.Levels; -import com.yahoo.system.CatchSigTerm; +import com.yahoo.yolean.system.CatchSignals; import java.io.IOException; import java.util.HashMap; @@ -150,8 +150,8 @@ public class Server implements Runnable { } } - private void setupSigTermHandler() { - CatchSigTerm.setup(signalCaught); // catch termination signal + private void setupSignalHandler() { + CatchSignals.setup(signalCaught); // catch termination and interrupt signals } private void waitForShutdown() { @@ -191,7 +191,7 @@ public class Server implements Runnable { String portString = System.getProperty(APPNAME + ".listenport", LISTEN_PORT); Server server = Server.getInstance(); - server.setupSigTermHandler(); + server.setupSignalHandler(); server.initialize(Integer.parseInt(portString)); Thread t = new Thread(server, "logserver main"); diff --git a/metrics/src/vespa/metrics/countmetricvalues.cpp b/metrics/src/vespa/metrics/countmetricvalues.cpp index 7fe3bb5b8ca..76ab2d4d83a 100644 --- a/metrics/src/vespa/metrics/countmetricvalues.cpp +++ b/metrics/src/vespa/metrics/countmetricvalues.cpp @@ -4,6 +4,6 @@ namespace metrics { -template class CountMetricValues<uint64_t>; +template struct CountMetricValues<uint64_t>; } // metrics diff --git a/metrics/src/vespa/metrics/memoryconsumption.h b/metrics/src/vespa/metrics/memoryconsumption.h index 1cbd8f1deab..9b2e0ec0254 100644 --- a/metrics/src/vespa/metrics/memoryconsumption.h +++ b/metrics/src/vespa/metrics/memoryconsumption.h @@ -21,8 +21,8 @@ namespace metrics { -class SeenStrings; -class SnapShotUsage; +struct SeenStrings; +struct SnapShotUsage; class MemoryConsumption : public vespalib::Printable { public: diff --git a/metrics/src/vespa/metrics/metric.h b/metrics/src/vespa/metrics/metric.h index d1313c3ca1a..85832ba08d1 100644 --- a/metrics/src/vespa/metrics/metric.h +++ b/metrics/src/vespa/metrics/metric.h @@ -8,8 +8,8 @@ namespace metrics { -class AbstractCountMetric; -class AbstractValueMetric; +struct AbstractCountMetric; +struct AbstractValueMetric; class Metric; class MetricSet; class MetricSnapshot; diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 4089236ca48..3b8de5927ec 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -274,8 +274,8 @@ public: private: void takeSnapshots(const MetricLockGuard &, time_t timeToProcess); - friend class MetricManagerTest; - friend class SnapshotTest; + friend struct MetricManagerTest; + friend struct SnapshotTest; void configure(const MetricLockGuard & guard, std::unique_ptr<MetricsmanagerConfig> conf); void run() override; diff --git a/metrics/src/vespa/metrics/summetric.h b/metrics/src/vespa/metrics/summetric.h index a9827bcca7d..c461fb25313 100644 --- a/metrics/src/vespa/metrics/summetric.h +++ b/metrics/src/vespa/metrics/summetric.h @@ -72,7 +72,7 @@ public: void addToSnapshot(Metric&, std::vector<Metric::UP> &) const override; private: - friend class MetricManagerTest; + friend struct MetricManagerTest; std::pair<std::vector<Metric::UP>, Metric::UP> generateSum() const; virtual void addTo(Metric&, std::vector<Metric::UP> *ownerList) const; diff --git a/metrics/src/vespa/metrics/valuemetricvalues.cpp b/metrics/src/vespa/metrics/valuemetricvalues.cpp index 12928db08dd..f309b07b1a4 100644 --- a/metrics/src/vespa/metrics/valuemetricvalues.cpp +++ b/metrics/src/vespa/metrics/valuemetricvalues.cpp @@ -3,10 +3,10 @@ namespace metrics { -template class ValueMetricValues<int64_t, int64_t>; +template struct ValueMetricValues<int64_t, int64_t>; template std::ostream & operator << (std::ostream & os, const ValueMetricValues<int64_t, int64_t> & v); -template class ValueMetricValues<double, double>; +template struct ValueMetricValues<double, double>; template std::ostream & operator << (std::ostream & os, const ValueMetricValues<double, double> & v); } // metrics diff --git a/processing/src/main/java/com/yahoo/processing/response/Data.java b/processing/src/main/java/com/yahoo/processing/response/Data.java index 2a631c3946d..fb8a296f0f3 100644 --- a/processing/src/main/java/com/yahoo/processing/response/Data.java +++ b/processing/src/main/java/com/yahoo/processing/response/Data.java @@ -17,6 +17,6 @@ public interface Data extends ListenableFreezable { /** * Returns the request that created this data */ - public Request request(); + Request request(); } diff --git a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h index 7ac2e701717..6dbf311ef82 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h +++ b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h @@ -8,7 +8,7 @@ #include <vector> namespace search { - class IDocumentWeightAttribute; + struct IDocumentWeightAttribute; class QueryTermSimple; } diff --git a/searchcore/src/vespa/searchcore/config/partitions.def b/searchcore/src/vespa/searchcore/config/partitions.def index 0b0a434b219..b9c3aabb37e 100644 --- a/searchcore/src/vespa/searchcore/config/partitions.def +++ b/searchcore/src/vespa/searchcore/config/partitions.def @@ -54,7 +54,7 @@ dataset[].mpp int default=1 dataset[].maxnodesdownperfixedrow int default=0 ## Use simple roundrobin or adaptive based on latency. -dataset[].useroundrobinforfixedrow bool default=false +dataset[].useroundrobinforfixedrow bool default=true ## specifies where a fdispatch or fsearch process can be contacted. ## must be in the format hostname:port/id where /id is optional. diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp index 574782c7cb5..ddbf3fe94c4 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp @@ -8,7 +8,7 @@ #include <vespa/searchlib/engine/packetconverter.h> #include <vespa/searchlib/engine/searchreply.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/xxhash/xxhash.h> +#include <xxhash.h> #include <vespa/log/log.h> LOG_SETUP(".fnet_search"); diff --git a/searchlib/src/vespa/searchlib/aggregation/aggregation.cpp b/searchlib/src/vespa/searchlib/aggregation/aggregation.cpp index 2c081328fd1..d40cdd5f13e 100644 --- a/searchlib/src/vespa/searchlib/aggregation/aggregation.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/aggregation.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/expression/resultvector.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/objects/visit.hpp> -#include <vespa/vespalib/xxhash/xxhash.h> +#include <xxhash.h> using namespace search::expression; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp index 3a8f1ad8de5..19076cc8db7 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp @@ -45,7 +45,7 @@ printEntry(vespalib::asciistream & os, const Entry & e) const float _asFloat; } u; u._asFloat = e.getValue(); - os << ", bvalue: 0x" << std::hex << u._asInt; + os << ", bvalue: 0x" << vespalib::hex << u._asInt; os << "}"; } @@ -65,7 +65,7 @@ printEntry(vespalib::asciistream & os, const Entry & e) const double _asDouble; } u; u._asDouble = e.getValue(); - os << ", bvalue: 0x" << std::hex << u._asLong; + os << ", bvalue: 0x" << vespalib::hex << u._asLong; os << "}"; } diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp index 32f02359162..8c32872bcec 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp @@ -2,8 +2,8 @@ #include "chunkformats.h" #include <vespa/vespalib/util/crc.h> -#include <vespa/vespalib/xxhash/xxhash.h> #include <vespa/vespalib/util/stringfmt.h> +#include <xxhash.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/value.cpp b/searchlib/src/vespa/searchlib/docstore/value.cpp index 8750413e3bc..68bb060683d 100644 --- a/searchlib/src/vespa/searchlib/docstore/value.cpp +++ b/searchlib/src/vespa/searchlib/docstore/value.cpp @@ -3,7 +3,7 @@ #include "value.h" #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/compressor.h> -#include <vespa/vespalib/xxhash/xxhash.h> +#include <xxhash.h> using vespalib::compression::compress; using vespalib::compression::decompress; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 47ab81f9b2e..126028c27e9 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -2,11 +2,11 @@ #include "domainpart.h" #include <vespa/vespalib/util/crc.h> -#include <vespa/vespalib/xxhash/xxhash.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/fastlib/io/bufferedfile.h> +#include <xxhash.h> #include <vespa/log/log.h> LOG_SETUP(".transactionlog.domainpart"); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index b9177ac8782..a95c8667b7d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -355,8 +355,21 @@ JuniperTeaserDFW::Init( return rc; } +vespalib::stringref +DynamicTeaserDFW::getJuniperInput(GeneralResult *gres, GetDocsumsState *state) { + int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue); + ResEntry *entry = gres->GetEntry(idx); + if (entry != nullptr) { + const char *buf; + uint32_t buflen; + entry->_resolve_field(&buf, &buflen, &state->_docSumFieldSpace); + return vespalib::stringref(buf, buflen); + } + return vespalib::stringref(); +} + vespalib::string -DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid, GeneralResult *gres, GetDocsumsState *state) +DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid, vespalib::stringref input, GetDocsumsState *state) { if (state->_dynteaser._query == nullptr) { JuniperQueryAdapter iq(state->_kwExtractor, @@ -384,29 +397,20 @@ DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid, GeneralResult *gres, GetDocs state->_dynteaser._config = _juniperConfig.get(); state->_dynteaser._result = nullptr; - int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue); - ResEntry *entry = gres->GetEntry(idx); - - if (entry != nullptr && state->_dynteaser._query != nullptr) { - - // obtain Juniper input - const char *buf; - uint32_t buflen; - - entry->_resolve_field(&buf, &buflen, &state->_docSumFieldSpace); + if (state->_dynteaser._query != nullptr) { if (LOG_WOULD_LOG(spam)) { std::ostringstream hexDump; - hexDump << vespalib::HexDump(buf, buflen); + hexDump << vespalib::HexDump(input.data(), input.length()); LOG(spam, "makeDynamicTeaser: docid=%d, input='%s', hexdump:\n%s", - docid, std::string(buf, buflen).c_str(), hexDump.str().c_str()); + docid, input.data(), hexDump.str().c_str()); } - uint32_t langid = static_cast<uint32_t>(-1); + auto langid = static_cast<uint32_t>(-1); state->_dynteaser._result = juniper::Analyse(_juniperConfig.get(), state->_dynteaser._query, - buf, buflen, docid, _inputFieldEnumValue, langid); + input.data(), input.length(), docid, _inputFieldEnumValue, langid); } } @@ -435,9 +439,12 @@ void DynamicTeaserDFW::insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) { - vespalib::string teaser = makeDynamicTeaser(docid, gres, state); - vespalib::Memory value(teaser.c_str(), teaser.size()); - target.insertString(value); + vespalib::stringref input = getJuniperInput(gres, state); + if (input.length() > 0) { + vespalib::string teaser = makeDynamicTeaser(docid, input, state); + vespalib::Memory value(teaser.c_str(), teaser.size()); + target.insertString(value); + } } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h index f6218d563ed..17beddf1e60 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h @@ -49,8 +49,9 @@ class DynamicTeaserDFW : public JuniperTeaserDFW public: DynamicTeaserDFW(juniper::Juniper * juniper) : JuniperTeaserDFW(juniper) { } + vespalib::stringref getJuniperInput(GeneralResult *gres, GetDocsumsState *state); vespalib::string makeDynamicTeaser(uint32_t docid, - GeneralResult *gres, + vespalib::stringref input, GetDocsumsState *state); void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java index 873baeae710..92f7afa121d 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java @@ -81,7 +81,9 @@ class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable { @Override public void close() { - monitors.values().forEach(HealthMonitor::close); - monitors.clear(); + synchronized (guard) { + monitors.values().forEach(HealthMonitor::close); + monitors.clear(); + } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java index f015ecadbfc..9f51e0692a0 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java @@ -14,9 +14,11 @@ import com.yahoo.vespa.service.monitor.ServiceId; import java.time.Duration; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * @author hakonhall @@ -67,7 +69,7 @@ public class StateV1HealthModel implements AutoCloseable { ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo); for (PortInfo portInfo : serviceInfo.getPorts()) { - if (portInfo.getTags().containsAll(HTTP_HEALTH_PORT_TAGS)) { + if (portTaggedWith(portInfo, HTTP_HEALTH_PORT_TAGS)) { // The host-admin-in-zone-application is one big hack. int port = isNodeAdmin ? HostAdminApplication.HOST_ADMIN_HEALT_PORT : portInfo.getPort(); StateV1HealthEndpoint endpoint = new StateV1HealthEndpoint( @@ -88,6 +90,18 @@ public class StateV1HealthModel implements AutoCloseable { return endpoints; } + static boolean portTaggedWith(PortInfo portInfo, List<String> requiredTags) { + // vespa-model-inspect displays upper case tags, while actual tags for (at least) node-admin are lower case. + Collection<String> upperCasePortTags = portInfo.getTags().stream().map(String::toUpperCase).collect(Collectors.toSet()); + for (var tag : requiredTags) { + if (!upperCasePortTags.contains(tag.toUpperCase())) { + return false; + } + } + + return true; + } + @Override public void close() { executor.close(); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java index fcbc3cd9b9f..9803fb75423 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.service.health; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ClusterId; @@ -23,7 +24,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -88,4 +91,16 @@ public class StateV1HealthModelTest { assertEquals(ZoneApplication.getNodeAdminClusterId(), serviceId.getClusterId()); assertEquals(ZoneApplication.getNodeAdminServiceType(), serviceId.getServiceType()); } + + @Test + public void caseInsensitiveTagMatching() { + PortInfo portInfo = mock(PortInfo.class); + when(portInfo.getTags()).thenReturn(List.of("http", "STATE", "foo")); + assertTrue(StateV1HealthModel.portTaggedWith(portInfo, StateV1HealthModel.HTTP_HEALTH_PORT_TAGS)); + assertTrue(StateV1HealthModel.portTaggedWith(portInfo, List.of("HTTP", "state"))); + + when(portInfo.getTags()).thenReturn(List.of("http", "foo")); + assertFalse(StateV1HealthModel.portTaggedWith(portInfo, StateV1HealthModel.HTTP_HEALTH_PORT_TAGS)); + assertFalse(StateV1HealthModel.portTaggedWith(portInfo, List.of("HTTP", "state"))); + } }
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/system/CatchSigTerm.java b/vespajlib/src/main/java/com/yahoo/system/CatchSigTerm.java deleted file mode 100644 index e1f037f5d98..00000000000 --- a/vespajlib/src/main/java/com/yahoo/system/CatchSigTerm.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.system; - -import java.lang.reflect.*; - -// import sun.misc.Signal; -// import sun.misc.SignalHandler; - -import java.util.concurrent.atomic.AtomicBoolean; - -public class CatchSigTerm { - /** - * Sets up a signal handler for SIGTERM, where a given AtomicBoolean - * gets a true value when the TERM signal is caught. - * - * Callers basically have two options for acting on the TERM signal: - * - * They may choose to synchronize and wait() on this variable, - * and they will be notified when it changes state to true. To avoid - * problems with spurious wakeups, use a while loop and wait() - * again if the state is still false. As soon as the caller has been - * woken up and the state is true, the application should exit as - * soon as possible. - * - * They may also choose to poll the state of this variable. As soon - * as its state becomes true, the signal has been received, and the - * application should exit as soon as possible. - * - * @param signalCaught set to false initially, will be set to true when SIGTERM is caught. - */ - @SuppressWarnings("rawtypes") - public static void setup(final AtomicBoolean signalCaught) { - signalCaught.set(false); - try { - Class shc = Class.forName("sun.misc.SignalHandler"); - Class ssc = Class.forName("sun.misc.Signal"); - - InvocationHandler ihandler = new InvocationHandler() { - public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - synchronized (signalCaught) { - signalCaught.set(true); - signalCaught.notifyAll(); - } - return null; - } - }; - Object shandler = Proxy.newProxyInstance(CatchSigTerm.class.getClassLoader(), - new Class[] { shc }, - ihandler); - Constructor[] c = ssc.getDeclaredConstructors(); - assert c.length == 1; - Object sigterm = c[0].newInstance("TERM"); - Method m = findMethod(ssc, "handle"); - assert m != null; // "NoSuchMethodException" - m.invoke(null, sigterm, shandler); - } catch (ClassNotFoundException | InvocationTargetException | InstantiationException | IllegalAccessException e) { - System.err.println("FAILED setting up signal catching: "+e); - } - } - - @SuppressWarnings("rawtypes") - private static Method findMethod(Class c, String name) { - for (Method m : c.getDeclaredMethods()) { - if (m.getName().equals(name)) { - return m; - } - } - return null; - } -} diff --git a/vespajlib/src/test/java/com/yahoo/system/CatchSigTermTestCase.java b/vespajlib/src/test/java/com/yahoo/system/CatchSigTermTestCase.java deleted file mode 100644 index 9b370d4ce10..00000000000 --- a/vespajlib/src/test/java/com/yahoo/system/CatchSigTermTestCase.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.system; - -import org.junit.Test; - -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * @author arnej27959 - */ -public class CatchSigTermTestCase { - - @Test - public void testThatSetupCompiles() { - CatchSigTerm.setup(new AtomicBoolean(false)); - } - -} diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 24f7d6e9214..a9d10954a6f 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -6,6 +6,7 @@ vespa_define_module( EXTERNAL_DEPENDS lz4 + xxhash zstd APPS @@ -145,5 +146,4 @@ vespa_define_module( src/vespa/vespalib/trace src/vespa/vespalib/util src/vespa/vespalib/websocket - src/vespa/vespalib/xxhash ) diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt index 676362a7aef..bab9fa79947 100644 --- a/vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/CMakeLists.txt @@ -21,7 +21,6 @@ vespa_add_library(vespalib $<TARGET_OBJECTS:vespalib_vespalib_trace> $<TARGET_OBJECTS:vespalib_vespalib_util> $<TARGET_OBJECTS:vespalib_vespalib_websocket> - $<TARGET_OBJECTS:vespalib_vespalib_xxhash> INSTALL lib64 DEPENDS gcc diff --git a/vespalib/src/vespa/vespalib/data/input_reader.h b/vespalib/src/vespa/vespalib/data/input_reader.h index f92ced7e508..41c42327b6c 100644 --- a/vespalib/src/vespa/vespalib/data/input_reader.h +++ b/vespalib/src/vespa/vespalib/data/input_reader.h @@ -8,7 +8,7 @@ namespace vespalib { -class Input; +struct Input; /** * A utility wrapper for the Input interface that supplies us with an diff --git a/vespalib/src/vespa/vespalib/data/output_writer.h b/vespalib/src/vespa/vespalib/data/output_writer.h index 70790fa6b5d..1e8afac0dfa 100644 --- a/vespalib/src/vespa/vespalib/data/output_writer.h +++ b/vespalib/src/vespa/vespalib/data/output_writer.h @@ -7,7 +7,7 @@ namespace vespalib { -class Output; +struct Output; /** * Thin layer on top of the Output interface that supplies us with an diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.h b/vespalib/src/vespa/vespalib/data/slime/inserter.h index dff37183ac7..e23c7ea7ec0 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inserter.h +++ b/vespalib/src/vespa/vespalib/data/slime/inserter.h @@ -13,7 +13,7 @@ class Slime; namespace slime { -class Cursor; +struct Cursor; //----------------------------------------------------------------------------- diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.h b/vespalib/src/vespa/vespalib/data/slime/json_format.h index f657f0439bf..dca77780e09 100644 --- a/vespalib/src/vespa/vespalib/data/slime/json_format.h +++ b/vespalib/src/vespa/vespalib/data/slime/json_format.h @@ -13,7 +13,7 @@ class Slime; namespace slime { -class Inspector; +struct Inspector; struct JsonFormat { static void encode(const Inspector &inspector, Output &output, bool compact); diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp index 6c2b60d15c1..bf875a70985 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp @@ -1,10 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "crypto_engine.h" -#include <vector> -#include <chrono> -#include <thread> -#include <vespa/vespalib/xxhash/xxhash.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/net/tls/authorization_mode.h> #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h> @@ -14,7 +10,11 @@ #include <vespa/vespalib/net/tls/tls_crypto_engine.h> #include <vespa/vespalib/net/tls/maybe_tls_crypto_engine.h> #include <vespa/vespalib/data/smart_buffer.h> -#include <assert.h> +#include <vector> +#include <chrono> +#include <thread> +#include <xxhash.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".vespalib.net.crypto_engine"); diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h index 246fd5cfc66..1eed1ae0612 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h @@ -47,7 +47,7 @@ struct DecodeResult { bool frame_decoded_ok() const noexcept { return (state == State::OK); } }; -class TlsContext; +struct TlsContext; // TODO move to different namespace, not dependent on TLS? diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h index 21581256ce6..3c491c2bd72 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h @@ -6,7 +6,7 @@ #include <vespa/vespalib/net/tls/crypto_codec.h> #include <memory> -namespace vespalib::net::tls { class TlsContext; } +namespace vespalib::net::tls { struct TlsContext; } namespace vespalib::net::tls::impl { diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h index 6e8fa368df7..e0effb1b52e 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h @@ -17,7 +17,7 @@ class FastOS_ThreadPool; namespace vespalib { -namespace thread { class ThreadInit; } +namespace thread { struct ThreadInit; } // Convenience macro used to create a function that can be used as an // init function when creating an executor to inject a frame with the diff --git a/vespalib/src/vespa/vespalib/websocket/handler.cpp b/vespalib/src/vespa/vespalib/websocket/handler.cpp index c2010c874dc..ba4a3307c1f 100644 --- a/vespalib/src/vespa/vespalib/websocket/handler.cpp +++ b/vespalib/src/vespa/vespalib/websocket/handler.cpp @@ -10,6 +10,6 @@ struct DummyItem {}; } // namespace vespalib::ws::<unnamed> -template class Handler<DummyItem>; +template struct Handler<DummyItem>; } // namespace vespalib::ws diff --git a/vespalib/src/vespa/vespalib/xxhash/CMakeLists.txt b/vespalib/src/vespa/vespalib/xxhash/CMakeLists.txt deleted file mode 100644 index 016cfe24325..00000000000 --- a/vespalib/src/vespa/vespalib/xxhash/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(vespalib_vespalib_xxhash OBJECT - SOURCES - xxhash.c - DEPENDS -) diff --git a/vespalib/src/vespa/vespalib/xxhash/xxhash.c b/vespalib/src/vespa/vespalib/xxhash/xxhash.c deleted file mode 100644 index 46b419795a2..00000000000 --- a/vespalib/src/vespa/vespalib/xxhash/xxhash.c +++ /dev/null @@ -1,935 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* -xxHash - Fast Hash algorithm -Copyright (C) 2012-2014, Yann Collet. -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -- public discussion board : https://groups.google.com/forum/#!forum/lz4c -*/ - - -//************************************** -// Tuning parameters -//************************************** -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. -// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. -// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). -#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_USE_UNALIGNED_ACCESS 1 -#endif - -// XXH_ACCEPT_NULL_INPUT_POINTER : -// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. -// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. -// This option has a very small performance cost (only measurable on small inputs). -// By default, this option is disabled. To enable it, uncomment below define : -// #define XXH_ACCEPT_NULL_INPUT_POINTER 1 - -// XXH_FORCE_NATIVE_FORMAT : -// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. -// Results are therefore identical for little-endian and big-endian CPU. -// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. -// Should endian-independance be of no importance for your application, you may set the #define below to 1. -// It will improve speed for Big-endian CPU. -// This option has no impact on Little_Endian CPU. -#define XXH_FORCE_NATIVE_FORMAT 0 - -//************************************** -// Compiler Specific Options -//************************************** -// Disable some Visual warning messages -#ifdef _MSC_VER // Visual Studio -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -#endif - -#ifdef _MSC_VER // Visual Studio -# define FORCE_INLINE static __forceinline -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - -//************************************** -// Includes & Memory related functions -//************************************** -#include "xxhash.h" -// Modify the local functions below should you wish to use some other memory routines -// for malloc(), free() -#include <stdlib.h> -FORCE_INLINE void* XXH_malloc(size_t s) -{ - return malloc(s); -} -FORCE_INLINE void XXH_free (void* p) -{ - free(p); -} -// for memcpy() -#include <string.h> -FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) -{ - return memcpy(dest,src,size); -} - - -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 -# include <stdint.h> -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED -#endif - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif - -typedef struct _U32_S -{ - U32 v; -} _PACKED U32_S; -typedef struct _U64_S -{ - U64 v; -} _PACKED U64_S; - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif - -#define A32(x) (((const U32_S *)(x))->v) -#define A64(x) (((const U64_S *)(x))->v) - - -//*************************************** -// Compiler-specific Functions and Macros -//*************************************** -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -// Note : although _rotl exists for minGW (GCC under windows), performance seems poor -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) -#endif - -#if defined(_MSC_VER) // Visual Studio -# define XXH_swap32 _byteswap_ulong -# define XXH_swap64 _byteswap_uint64 -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -# define XXH_swap64 __builtin_bswap64 -#else -static inline U32 XXH_swap32 (U32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -static inline U64 XXH_swap64 (U64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); -} -#endif - - -//************************************** -// Constants -//************************************** -#define PRIME32_1 2654435761U -#define PRIME32_2 2246822519U -#define PRIME32_3 3266489917U -#define PRIME32_4 668265263U -#define PRIME32_5 374761393U - -#define PRIME64_1 11400714785074694791ULL -#define PRIME64_2 14029467366897019727ULL -#define PRIME64_3 1609587929392839161ULL -#define PRIME64_4 9650029242287828579ULL -#define PRIME64_5 2870177450012600261ULL - -//************************************** -// Architecture Macros -//************************************** -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; -#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch -static const int one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one)) -#endif - - -//************************************** -// Macros -//************************************** -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations - - -//**************************** -// Memory reads -//**************************** -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); - else - return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); -} - -FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} - -FORCE_INLINE U64 XXH_readLE64_align(const U64* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr)); - else - return endian==XXH_littleEndian ? *ptr : XXH_swap64(*ptr); -} - -FORCE_INLINE U64 XXH_readLE64(const U64* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} - - -//**************************** -// Simple Hash Functions -//**************************** -FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align((const U32*)p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) - { - len=0; - bEnd=p=(const BYTE*)(size_t)16; - } -#endif - - if (len>=16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do - { - v1 += XXH_get32bits(p) * PRIME32_2; - v1 = XXH_rotl32(v1, 13); - v1 *= PRIME32_1; - p+=4; - v2 += XXH_get32bits(p) * PRIME32_2; - v2 = XXH_rotl32(v2, 13); - v2 *= PRIME32_1; - p+=4; - v3 += XXH_get32bits(p) * PRIME32_2; - v3 = XXH_rotl32(v3, 13); - v3 *= PRIME32_1; - p+=4; - v4 += XXH_get32bits(p) * PRIME32_2; - v4 = XXH_rotl32(v4, 13); - v4 *= PRIME32_1; - p+=4; - } - while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } - else - { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p+4<=bEnd) - { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p<bEnd) - { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -unsigned int XXH32 (const void* input, size_t len, unsigned seed) -{ -#if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs - XXH32_state_t state; - XXH32_reset(&state, seed); - XXH32_update(&state, input, len); - return XXH32_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - -# if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage - { - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } -# endif - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - -FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align((const U64*)p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) - { - len=0; - bEnd=p=(const BYTE*)(size_t)32; - } -#endif - - if (len>=32) - { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; - - do - { - v1 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - v2 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - v3 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - v4 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - } - while (p<=limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - - v1 *= PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - h64 ^= v1; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v2 *= PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - h64 ^= v2; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v3 *= PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - h64 ^= v3; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v4 *= PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - h64 ^= v4; - h64 = h64 * PRIME64_1 + PRIME64_4; - } - else - { - h64 = seed + PRIME64_5; - } - - h64 += (U64) len; - - while (p+8<=bEnd) - { - U64 k1 = XXH_get64bits(p); - k1 *= PRIME64_2; - k1 = XXH_rotl64(k1,31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) - { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p<bEnd) - { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) -{ -#if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs - XXH64_state_t state; - XXH64_reset(&state, seed); - XXH64_update(&state, input, len); - return XXH64_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - -# if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage - { - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } -# endif - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - -/**************************************************** - * Advanced Hash Functions -****************************************************/ - -/*** Allocation ***/ -typedef struct -{ - U64 total_len; - U32 seed; - U32 v1; - U32 v2; - U32 v3; - U32 v4; - U32 memsize; - char memory[16]; -} XXH_istate32_t; - -typedef struct -{ - U64 total_len; - U64 seed; - U64 v1; - U64 v2; - U64 v3; - U64 v4; - U32 memsize; - char memory[32]; -} XXH_istate64_t; - - -XXH32_state_t* XXH32_createState(void) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); // A compilation error here means XXH32_state_t is not large enough - return (XXH32_state_t*)malloc(sizeof(XXH32_state_t)); -} -XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - free(statePtr); - return XXH_OK; -}; - -XXH64_state_t* XXH64_createState(void) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); // A compilation error here means XXH64_state_t is not large enough - return (XXH64_state_t*)malloc(sizeof(XXH64_state_t)); -} -XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) -{ - free(statePtr); - return XXH_OK; -}; - - -/*** Hash feed ***/ - -XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed) -{ - XXH_istate32_t* state = (XXH_istate32_t*) state_in; - state->seed = seed; - state->v1 = seed + PRIME32_1 + PRIME32_2; - state->v2 = seed + PRIME32_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME32_1; - state->total_len = 0; - state->memsize = 0; - return XXH_OK; -} - -XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed) -{ - XXH_istate64_t* state = (XXH_istate64_t*) state_in; - state->seed = seed; - state->v1 = seed + PRIME64_1 + PRIME64_2; - state->v2 = seed + PRIME64_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME64_1; - state->total_len = 0; - state->memsize = 0; - return XXH_OK; -} - - -FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian) -{ - XXH_istate32_t* state = (XXH_istate32_t *) state_in; - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 16) // fill in tmp buffer - { - XXH_memcpy(state->memory + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) // some data left from previous update - { - XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); - { - const U32* p32 = (const U32*)state->memory; - state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v1 = XXH_rotl32(state->v1, 13); - state->v1 *= PRIME32_1; - p32++; - state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v2 = XXH_rotl32(state->v2, 13); - state->v2 *= PRIME32_1; - p32++; - state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v3 = XXH_rotl32(state->v3, 13); - state->v3 *= PRIME32_1; - p32++; - state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v4 = XXH_rotl32(state->v4, 13); - state->v4 *= PRIME32_1; - p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do - { - v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; - v1 = XXH_rotl32(v1, 13); - v1 *= PRIME32_1; - p+=4; - v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; - v2 = XXH_rotl32(v2, 13); - v2 *= PRIME32_1; - p+=4; - v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; - v3 = XXH_rotl32(v3, 13); - v3 *= PRIME32_1; - p+=4; - v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; - v4 = XXH_rotl32(v4, 13); - v4 *= PRIME32_1; - p+=4; - } - while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) - { - XXH_memcpy(state->memory, p, bEnd-p); - state->memsize = (int)(bEnd-p); - } - - return XXH_OK; -} - -XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) -{ - const XXH_istate32_t* state = (const XXH_istate32_t*) state_in; - const BYTE * p = (const BYTE*)state->memory; - const BYTE* bEnd = (const BYTE*)state->memory + state->memsize; - U32 h32; - - if (state->total_len >= 16) - { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } - else - { - h32 = state->seed + PRIME32_5; - } - - h32 += (U32) state->total_len; - - while (p+4<=bEnd) - { - h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p<bEnd) - { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -U32 XXH32_digest (const XXH32_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); -} - - -FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian) -{ - XXH_istate64_t * state = (XXH_istate64_t *) state_in; - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 32) // fill in tmp buffer - { - XXH_memcpy(state->memory + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) // some data left from previous update - { - XXH_memcpy(state->memory + state->memsize, input, 32-state->memsize); - { - const U64* p64 = (const U64*)state->memory; - state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v1 = XXH_rotl64(state->v1, 31); - state->v1 *= PRIME64_1; - p64++; - state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v2 = XXH_rotl64(state->v2, 31); - state->v2 *= PRIME64_1; - p64++; - state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v3 = XXH_rotl64(state->v3, 31); - state->v3 *= PRIME64_1; - p64++; - state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v4 = XXH_rotl64(state->v4, 31); - state->v4 *= PRIME64_1; - p64++; - } - p += 32-state->memsize; - state->memsize = 0; - } - - if (p+32 <= bEnd) - { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - do - { - v1 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - p+=8; - v2 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - p+=8; - v3 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - p+=8; - v4 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - p+=8; - } - while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) - { - XXH_memcpy(state->memory, p, bEnd-p); - state->memsize = (int)(bEnd-p); - } - - return XXH_OK; -} - -XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) -{ - const XXH_istate64_t * state = (const XXH_istate64_t *) state_in; - const BYTE * p = (const BYTE*)state->memory; - const BYTE* bEnd = (const BYTE*)state->memory + state->memsize; - U64 h64; - - if (state->total_len >= 32) - { - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - - v1 *= PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - h64 ^= v1; - h64 = h64*PRIME64_1 + PRIME64_4; - - v2 *= PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - h64 ^= v2; - h64 = h64*PRIME64_1 + PRIME64_4; - - v3 *= PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - h64 ^= v3; - h64 = h64*PRIME64_1 + PRIME64_4; - - v4 *= PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - h64 ^= v4; - h64 = h64*PRIME64_1 + PRIME64_4; - } - else - { - h64 = state->seed + PRIME64_5; - } - - h64 += (U64) state->total_len; - - while (p+8<=bEnd) - { - U64 k1 = XXH_readLE64((const U64*)p, endian); - k1 *= PRIME64_2; - k1 = XXH_rotl64(k1,31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) - { - h64 ^= (U64)(XXH_readLE32((const U32*)p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p<bEnd) - { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -unsigned long long XXH64_digest (const XXH64_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); -} - - diff --git a/vespalib/src/vespa/vespalib/xxhash/xxhash.h b/vespalib/src/vespa/vespalib/xxhash/xxhash.h deleted file mode 100644 index dfd17b76d77..00000000000 --- a/vespalib/src/vespa/vespalib/xxhash/xxhash.h +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - xxHash - Extremely Fast Hash algorithm - Header File - Copyright (C) 2012-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash/ -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. -*/ - -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - - -/***************************** - Includes -*****************************/ -#include <stddef.h> /* size_t */ - - -/***************************** - Type -*****************************/ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - - -/***************************** - Simple Hash Functions -*****************************/ - -unsigned int XXH32 (const void* input, size_t length, unsigned seed); -unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); - -/* -XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". - The memory between input & input+length must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - This function successfully passes all SMHasher tests. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s -XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". -*/ - - - -/***************************** - Advanced Hash Functions -*****************************/ -typedef struct { long long ll[ 6]; } XXH32_state_t; -typedef struct { long long ll[11]; } XXH64_state_t; - -/* -These structures allow static allocation of XXH states. -States must then be initialized using XXHnn_reset() before first use. - -If you prefer dynamic allocation, please refer to functions below. -*/ - -XXH32_state_t* XXH32_createState(void); -XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); - -XXH64_state_t* XXH64_createState(void); -XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); - -/* -These functions create and release memory for XXH state. -States must then be initialized using XXHnn_reset() before first use. -*/ - - -XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed); -XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -unsigned int XXH32_digest (const XXH32_state_t* statePtr); - -XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -unsigned long long XXH64_digest (const XXH64_state_t* statePtr); - -/* -These functions calculate the xxHash of an input provided in multiple smaller packets, -as opposed to an input provided as a single block. - -XXH state space must first be allocated, using either static or dynamic method provided above. - -Start a new hash by initializing state with a seed, using XXHnn_reset(). - -Then, feed the hash state by calling XXHnn_update() as many times as necessary. -Obviously, input must be valid, meaning allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - -Finally, you can produce a hash anytime, by using XXHnn_digest(). -This function returns the final nn-bits hash. -You can nonetheless continue feeding the hash state with more input, -and therefore get some new hashes, by calling again XXHnn_digest(). - -When you are done, don't forget to free XXH state space, using typically XXHnn_freeState(). -*/ - - -#if defined (__cplusplus) -} -#endif diff --git a/yolean/src/main/java/com/yahoo/yolean/system/CatchSigTerm.java b/yolean/src/main/java/com/yahoo/yolean/system/CatchSignals.java index ee87ab9c081..dce70fbbb5f 100644 --- a/yolean/src/main/java/com/yahoo/yolean/system/CatchSigTerm.java +++ b/yolean/src/main/java/com/yahoo/yolean/system/CatchSignals.java @@ -8,12 +8,12 @@ import java.lang.reflect.*; import java.util.concurrent.atomic.AtomicBoolean; -public class CatchSigTerm { +public class CatchSignals { /** - * Sets up a signal handler for SIGTERM, where a given AtomicBoolean - * gets a true value when the TERM signal is caught. + * Sets up a signal handler for SIGTERM and SIGINT, where a given AtomicBoolean + * gets a true value when the signal is caught. * - * Callers basically have two options for acting on the TERM signal: + * Callers basically have two options for acting on the signal: * * They may choose to synchronize and wait() on this variable, * and they will be notified when it changes state to true. To avoid @@ -26,7 +26,7 @@ public class CatchSigTerm { * as its state becomes true, the signal has been received, and the * application should exit as soon as possible. * - * @param signalCaught set to false initially, will be set to true when SIGTERM is caught. + * @param signalCaught set to false initially, will be set to true when SIGTERM or SIGINT is caught. */ @SuppressWarnings("rawtypes") public static void setup(final AtomicBoolean signalCaught) { @@ -44,15 +44,17 @@ public class CatchSigTerm { return null; } }; - Object shandler = Proxy.newProxyInstance(CatchSigTerm.class.getClassLoader(), + Object shandler = Proxy.newProxyInstance(CatchSignals.class.getClassLoader(), new Class[] { shc }, ihandler); Constructor[] c = ssc.getDeclaredConstructors(); assert c.length == 1; Object sigterm = c[0].newInstance("TERM"); + Object sigint = c[0].newInstance("INT"); Method m = findMethod(ssc, "handle"); assert m != null; // "NoSuchMethodException" m.invoke(null, sigterm, shandler); + m.invoke(null, sigint, shandler); } catch (ClassNotFoundException | InvocationTargetException | InstantiationException | IllegalAccessException e) { System.err.println("FAILED setting up signal catching: "+e); } diff --git a/yolean/src/test/java/com/yahoo/yolean/system/CatchSigTermTestCase.java b/yolean/src/test/java/com/yahoo/yolean/system/CatchSignalsTestCase.java index 056db2d3592..e68ce334f31 100644 --- a/yolean/src/test/java/com/yahoo/yolean/system/CatchSigTermTestCase.java +++ b/yolean/src/test/java/com/yahoo/yolean/system/CatchSignalsTestCase.java @@ -8,11 +8,11 @@ import java.util.concurrent.atomic.AtomicBoolean; /** * @author arnej27959 */ -public class CatchSigTermTestCase { +public class CatchSignalsTestCase { @Test public void testThatSetupCompiles() { - CatchSigTerm.setup(new AtomicBoolean(false)); + CatchSignals.setup(new AtomicBoolean(false)); } } |